2024-12-05 23:19:27,017 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1972e513 2024-12-05 23:19:27,057 main DEBUG Took 0.029174 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-05 23:19:27,068 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-05 23:19:27,069 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-05 23:19:27,074 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-05 23:19:27,076 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 23:19:27,116 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-05 23:19:27,230 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 23:19:27,242 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 23:19:27,244 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 23:19:27,246 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 23:19:27,253 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 23:19:27,255 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 23:19:27,256 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 23:19:27,256 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 23:19:27,257 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 23:19:27,257 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 23:19:27,269 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 23:19:27,272 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 23:19:27,274 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 23:19:27,277 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 23:19:27,282 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 23:19:27,283 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 23:19:27,295 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 23:19:27,298 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 23:19:27,299 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 23:19:27,300 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 23:19:27,309 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 23:19:27,318 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 23:19:27,333 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 23:19:27,333 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-05 23:19:27,335 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 23:19:27,335 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-05 23:19:27,352 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-05 23:19:27,358 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-05 23:19:27,367 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-05 23:19:27,374 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-05 23:19:27,378 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-05 23:19:27,379 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-05 23:19:27,391 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-05 23:19:27,394 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-05 23:19:27,397 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-05 23:19:27,397 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-05 23:19:27,398 main DEBUG createAppenders(={Console}) 2024-12-05 23:19:27,401 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1972e513 initialized 2024-12-05 23:19:27,401 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1972e513 2024-12-05 23:19:27,402 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@1972e513 OK. 2024-12-05 23:19:27,402 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-05 23:19:27,403 main DEBUG OutputStream closed 2024-12-05 23:19:27,404 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-05 23:19:27,405 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-05 23:19:27,405 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@5fbdfdcf OK 2024-12-05 23:19:27,670 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-05 23:19:27,673 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-05 23:19:27,674 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-05 23:19:27,675 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-05 23:19:27,676 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-05 23:19:27,676 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-05 23:19:27,677 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-05 23:19:27,677 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-05 23:19:27,678 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-05 23:19:27,678 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-05 23:19:27,678 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-05 23:19:27,679 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-05 23:19:27,680 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-05 23:19:27,681 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-05 23:19:27,681 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-05 23:19:27,681 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-05 23:19:27,682 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-05 23:19:27,682 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-05 23:19:27,685 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05 23:19:27,686 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@6ce86ce1) with optional ClassLoader: null 2024-12-05 23:19:27,686 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-05 23:19:27,687 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@6ce86ce1] started OK. 2024-12-05T23:19:28,404 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da 2024-12-05 23:19:28,411 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-05 23:19:28,416 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-05T23:19:28,484 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestAsyncReplicationAdminApi timeout: 26 mins 2024-12-05T23:19:28,556 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-05T23:19:28,561 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/cluster_ea30cb3b-51ca-5a5b-9a30-c0d33a20f2ca, deleteOnExit=true 2024-12-05T23:19:28,561 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-12-05T23:19:28,562 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/test.cache.data in system properties and HBase conf 2024-12-05T23:19:28,566 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/hadoop.tmp.dir in system properties and HBase conf 2024-12-05T23:19:28,573 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/hadoop.log.dir in system properties and HBase conf 2024-12-05T23:19:28,580 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-05T23:19:28,581 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-05T23:19:28,581 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-12-05T23:19:28,832 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-05T23:19:29,107 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-05T23:19:29,147 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-05T23:19:29,148 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-05T23:19:29,149 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-05T23:19:29,149 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T23:19:29,150 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-05T23:19:29,150 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-05T23:19:29,151 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-05T23:19:29,152 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T23:19:29,156 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-05T23:19:29,157 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/nfs.dump.dir in system properties and HBase conf 2024-12-05T23:19:29,158 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/java.io.tmpdir in system properties and HBase conf 2024-12-05T23:19:29,158 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-05T23:19:29,159 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-05T23:19:29,159 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-05T23:19:30,842 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-05T23:19:31,009 INFO [Time-limited test {}] log.Log(170): Logging initialized @6193ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-05T23:19:31,172 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T23:19:31,355 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T23:19:31,428 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T23:19:31,429 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T23:19:31,431 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-05T23:19:31,458 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T23:19:31,462 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1d9ceadd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/hadoop.log.dir/,AVAILABLE} 2024-12-05T23:19:31,463 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@f2948e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T23:19:31,783 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2b5eba6b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/java.io.tmpdir/jetty-localhost-42949-hadoop-hdfs-3_4_1-tests_jar-_-any-522090719704299194/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T23:19:31,820 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@79c1d5da{HTTP/1.1, (http/1.1)}{localhost:42949} 2024-12-05T23:19:31,821 INFO [Time-limited test {}] server.Server(415): Started @7006ms 2024-12-05T23:19:32,683 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-05T23:19:32,697 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-05T23:19:32,719 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-05T23:19:32,720 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-05T23:19:32,720 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-12-05T23:19:32,723 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@a00c17f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/hadoop.log.dir/,AVAILABLE} 2024-12-05T23:19:32,724 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@48604e1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-05T23:19:32,978 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@6e50d5cf{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/java.io.tmpdir/jetty-localhost-38867-hadoop-hdfs-3_4_1-tests_jar-_-any-3977413695397125275/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T23:19:32,979 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@291745d8{HTTP/1.1, (http/1.1)}{localhost:38867} 2024-12-05T23:19:32,979 INFO [Time-limited test {}] server.Server(415): Started @8165ms 2024-12-05T23:19:33,078 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-05T23:19:34,210 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/cluster_ea30cb3b-51ca-5a5b-9a30-c0d33a20f2ca/dfs/data/data2/current/BP-81780034-172.17.0.2-1733440770420/current, will proceed with Du for space computation calculation, 2024-12-05T23:19:34,215 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/cluster_ea30cb3b-51ca-5a5b-9a30-c0d33a20f2ca/dfs/data/data1/current/BP-81780034-172.17.0.2-1733440770420/current, will proceed with Du for space computation calculation, 2024-12-05T23:19:34,363 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-05T23:19:34,478 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe641acae44b906e5 with lease ID 0xad99326138475c88: Processing first storage report for DS-72ec784f-32f4-4aee-99be-58799187a955 from datanode DatanodeRegistration(127.0.0.1:38377, datanodeUuid=d04413b2-2858-4c3a-905b-f46e3d9f905c, infoPort=41975, infoSecurePort=0, ipcPort=38995, storageInfo=lv=-57;cid=testClusterID;nsid=403071319;c=1733440770420) 2024-12-05T23:19:34,480 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe641acae44b906e5 with lease ID 0xad99326138475c88: from storage DS-72ec784f-32f4-4aee-99be-58799187a955 node DatanodeRegistration(127.0.0.1:38377, datanodeUuid=d04413b2-2858-4c3a-905b-f46e3d9f905c, infoPort=41975, infoSecurePort=0, ipcPort=38995, storageInfo=lv=-57;cid=testClusterID;nsid=403071319;c=1733440770420), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-05T23:19:34,480 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xe641acae44b906e5 with lease ID 0xad99326138475c88: Processing first storage report for DS-e6e2c195-a964-4163-b451-a256e461545a from datanode DatanodeRegistration(127.0.0.1:38377, datanodeUuid=d04413b2-2858-4c3a-905b-f46e3d9f905c, infoPort=41975, infoSecurePort=0, ipcPort=38995, storageInfo=lv=-57;cid=testClusterID;nsid=403071319;c=1733440770420) 2024-12-05T23:19:34,481 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xe641acae44b906e5 with lease ID 0xad99326138475c88: from storage DS-e6e2c195-a964-4163-b451-a256e461545a node DatanodeRegistration(127.0.0.1:38377, datanodeUuid=d04413b2-2858-4c3a-905b-f46e3d9f905c, infoPort=41975, infoSecurePort=0, ipcPort=38995, storageInfo=lv=-57;cid=testClusterID;nsid=403071319;c=1733440770420), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-05T23:19:34,539 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da 2024-12-05T23:19:34,745 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/cluster_ea30cb3b-51ca-5a5b-9a30-c0d33a20f2ca/zookeeper_0, clientPort=50903, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/cluster_ea30cb3b-51ca-5a5b-9a30-c0d33a20f2ca/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/cluster_ea30cb3b-51ca-5a5b-9a30-c0d33a20f2ca/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-05T23:19:34,771 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=50903 2024-12-05T23:19:34,791 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T23:19:34,795 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T23:19:35,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741825_1001 (size=7) 2024-12-05T23:19:35,738 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9 with version=8 2024-12-05T23:19:35,738 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/hbase-staging 2024-12-05T23:19:35,921 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-05T23:19:36,371 INFO [Time-limited test {}] client.ConnectionUtils(129): master/9e5740704f6b:0 server-side Connection retries=6 2024-12-05T23:19:36,401 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T23:19:36,404 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T23:19:36,404 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T23:19:36,405 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T23:19:36,405 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T23:19:36,633 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T23:19:36,763 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-05T23:19:36,781 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-05T23:19:36,787 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T23:19:36,826 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 7431 (auto-detected) 2024-12-05T23:19:36,828 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-05T23:19:36,860 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:38841 2024-12-05T23:19:36,871 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T23:19:36,875 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T23:19:36,893 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:38841 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:36,983 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:388410x0, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:36,992 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:38841-0x10184ed0ca40000 connected 2024-12-05T23:19:37,038 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T23:19:37,042 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T23:19:37,047 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T23:19:37,054 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=38841 2024-12-05T23:19:37,055 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=38841 2024-12-05T23:19:37,055 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=38841 2024-12-05T23:19:37,056 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=38841 2024-12-05T23:19:37,056 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=38841 2024-12-05T23:19:37,076 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9, hbase.cluster.distributed=false 2024-12-05T23:19:37,232 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/9e5740704f6b:0 server-side Connection retries=6 2024-12-05T23:19:37,232 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T23:19:37,235 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-05T23:19:37,235 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-05T23:19:37,235 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-05T23:19:37,235 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-05T23:19:37,239 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-05T23:19:37,242 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-05T23:19:37,244 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:36187 2024-12-05T23:19:37,246 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-05T23:19:37,297 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-05T23:19:37,306 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T23:19:37,313 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T23:19:37,332 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:36187 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:37,350 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:361870x0, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:37,356 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:361870x0, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T23:19:37,369 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:36187-0x10184ed0ca40001 connected 2024-12-05T23:19:37,377 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T23:19:37,393 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-05T23:19:37,409 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36187 2024-12-05T23:19:37,425 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36187 2024-12-05T23:19:37,433 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36187 2024-12-05T23:19:37,470 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36187 2024-12-05T23:19:37,473 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36187 2024-12-05T23:19:37,504 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/9e5740704f6b,38841,1733440775912 2024-12-05T23:19:37,523 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T23:19:37,525 DEBUG [M:0;9e5740704f6b:38841 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9e5740704f6b:38841 2024-12-05T23:19:37,526 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T23:19:37,529 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9e5740704f6b,38841,1733440775912 2024-12-05T23:19:37,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T23:19:37,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-05T23:19:37,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T23:19:37,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T23:19:37,598 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T23:19:37,600 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-05T23:19:37,600 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9e5740704f6b,38841,1733440775912 from backup master directory 2024-12-05T23:19:37,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T23:19:37,612 WARN [master/9e5740704f6b:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T23:19:37,612 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9e5740704f6b,38841,1733440775912 2024-12-05T23:19:37,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9e5740704f6b,38841,1733440775912 2024-12-05T23:19:37,613 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-05T23:19:37,615 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-05T23:19:37,625 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-05T23:19:37,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741826_1002 (size=42) 2024-12-05T23:19:38,246 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/hbase.id with ID: a6e07b25-fe39-42c3-aefd-b0129d0da2f3 2024-12-05T23:19:38,357 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-05T23:19:38,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T23:19:38,449 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T23:19:38,533 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741827_1003 (size=196) 2024-12-05T23:19:38,574 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-05T23:19:38,579 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-05T23:19:38,620 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:402) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:973) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2470) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:590) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:587) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:38,630 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T23:19:38,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741828_1004 (size=1189) 2024-12-05T23:19:38,774 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/data/master/store 2024-12-05T23:19:38,816 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741829_1005 (size=34) 2024-12-05T23:19:39,223 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-05T23:19:39,224 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T23:19:39,226 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T23:19:39,226 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T23:19:39,226 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T23:19:39,226 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T23:19:39,227 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T23:19:39,227 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T23:19:39,227 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-05T23:19:39,230 WARN [master/9e5740704f6b:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/data/master/store/.initializing 2024-12-05T23:19:39,230 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/WALs/9e5740704f6b,38841,1733440775912 2024-12-05T23:19:39,239 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T23:19:39,254 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9e5740704f6b%2C38841%2C1733440775912, suffix=, logDir=hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/WALs/9e5740704f6b,38841,1733440775912, archiveDir=hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/oldWALs, maxLogs=10 2024-12-05T23:19:39,285 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(617): When create output stream for /user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/WALs/9e5740704f6b,38841,1733440775912/9e5740704f6b%2C38841%2C1733440775912.1733440779261, exclude list is [], retry=0 2024-12-05T23:19:39,306 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38377,DS-72ec784f-32f4-4aee-99be-58799187a955,DISK] 2024-12-05T23:19:39,310 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-05T23:19:39,364 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/WALs/9e5740704f6b,38841,1733440775912/9e5740704f6b%2C38841%2C1733440775912.1733440779261 2024-12-05T23:19:39,369 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41975:41975)] 2024-12-05T23:19:39,370 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-05T23:19:39,370 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T23:19:39,375 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T23:19:39,377 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T23:19:39,460 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T23:19:39,511 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-05T23:19:39,517 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T23:19:39,523 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T23:19:39,528 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T23:19:39,537 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-05T23:19:39,537 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T23:19:39,539 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T23:19:39,540 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T23:19:39,548 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-05T23:19:39,548 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T23:19:39,550 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T23:19:39,551 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-05T23:19:39,558 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-05T23:19:39,558 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T23:19:39,560 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T23:19:39,566 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T23:19:39,568 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-05T23:19:39,593 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-05T23:19:39,600 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-05T23:19:39,608 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T23:19:39,609 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65360181, jitterRate=-0.0260574072599411}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-05T23:19:39,616 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-05T23:19:39,618 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-05T23:19:39,673 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d8e72e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:19:39,720 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-12-05T23:19:39,740 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-05T23:19:39,740 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-05T23:19:39,743 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-05T23:19:39,745 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 1 msec 2024-12-05T23:19:39,752 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 6 msec 2024-12-05T23:19:39,753 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-05T23:19:39,801 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-05T23:19:39,819 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-05T23:19:39,822 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-12-05T23:19:39,826 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-05T23:19:39,828 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-05T23:19:39,831 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-12-05T23:19:39,834 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-05T23:19:39,842 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-05T23:19:39,845 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-12-05T23:19:39,849 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-05T23:19:39,854 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-12-05T23:19:39,880 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-05T23:19:39,883 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-05T23:19:39,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T23:19:39,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-05T23:19:39,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T23:19:39,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T23:19:39,904 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=9e5740704f6b,38841,1733440775912, sessionid=0x10184ed0ca40000, setting cluster-up flag (Was=false) 2024-12-05T23:19:39,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T23:19:39,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T23:19:39,939 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-05T23:19:39,942 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9e5740704f6b,38841,1733440775912 2024-12-05T23:19:39,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T23:19:39,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T23:19:39,962 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-05T23:19:39,965 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9e5740704f6b,38841,1733440775912 2024-12-05T23:19:40,059 DEBUG [RS:0;9e5740704f6b:36187 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9e5740704f6b:36187 2024-12-05T23:19:40,062 INFO [RS:0;9e5740704f6b:36187 {}] regionserver.HRegionServer(1008): ClusterId : a6e07b25-fe39-42c3-aefd-b0129d0da2f3 2024-12-05T23:19:40,093 DEBUG [RS:0;9e5740704f6b:36187 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-05T23:19:40,105 DEBUG [RS:0;9e5740704f6b:36187 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-05T23:19:40,105 DEBUG [RS:0;9e5740704f6b:36187 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-05T23:19:40,117 DEBUG [RS:0;9e5740704f6b:36187 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-05T23:19:40,118 DEBUG [RS:0;9e5740704f6b:36187 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@479cb87b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:19:40,130 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-12-05T23:19:40,133 DEBUG [RS:0;9e5740704f6b:36187 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a9be34e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9e5740704f6b/172.17.0.2:0 2024-12-05T23:19:40,137 INFO [RS:0;9e5740704f6b:36187 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-12-05T23:19:40,138 INFO [RS:0;9e5740704f6b:36187 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-12-05T23:19:40,138 DEBUG [RS:0;9e5740704f6b:36187 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-12-05T23:19:40,143 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-12-05T23:19:40,145 INFO [RS:0;9e5740704f6b:36187 {}] regionserver.HRegionServer(3073): reportForDuty to master=9e5740704f6b,38841,1733440775912 with isa=9e5740704f6b/172.17.0.2:36187, startcode=1733440777223 2024-12-05T23:19:40,148 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-05T23:19:40,159 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9e5740704f6b,38841,1733440775912 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-05T23:19:40,164 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9e5740704f6b:0, corePoolSize=5, maxPoolSize=5 2024-12-05T23:19:40,164 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9e5740704f6b:0, corePoolSize=5, maxPoolSize=5 2024-12-05T23:19:40,164 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9e5740704f6b:0, corePoolSize=5, maxPoolSize=5 2024-12-05T23:19:40,164 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9e5740704f6b:0, corePoolSize=5, maxPoolSize=5 2024-12-05T23:19:40,165 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9e5740704f6b:0, corePoolSize=10, maxPoolSize=10 2024-12-05T23:19:40,165 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9e5740704f6b:0, corePoolSize=1, maxPoolSize=1 2024-12-05T23:19:40,167 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9e5740704f6b:0, corePoolSize=2, maxPoolSize=2 2024-12-05T23:19:40,167 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9e5740704f6b:0, corePoolSize=1, maxPoolSize=1 2024-12-05T23:19:40,181 DEBUG [RS:0;9e5740704f6b:36187 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T23:19:40,190 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733440810185 2024-12-05T23:19:40,194 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-05T23:19:40,197 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-12-05T23:19:40,197 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-05T23:19:40,197 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-12-05T23:19:40,201 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-05T23:19:40,203 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-05T23:19:40,203 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-05T23:19:40,203 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-05T23:19:40,207 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T23:19:40,207 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T23:19:40,223 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T23:19:40,245 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-05T23:19:40,247 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-05T23:19:40,247 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-05T23:19:40,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741831_1007 (size=1039) 2024-12-05T23:19:40,288 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-12-05T23:19:40,289 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9 2024-12-05T23:19:40,291 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-05T23:19:40,292 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-05T23:19:40,300 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9e5740704f6b:0:becomeActiveMaster-HFileCleaner.large.0-1733440780294,5,FailOnTimeoutGroup] 2024-12-05T23:19:40,302 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9e5740704f6b:0:becomeActiveMaster-HFileCleaner.small.0-1733440780301,5,FailOnTimeoutGroup] 2024-12-05T23:19:40,303 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-05T23:19:40,303 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-05T23:19:40,304 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-05T23:19:40,305 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-05T23:19:40,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741832_1008 (size=32) 2024-12-05T23:19:40,363 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T23:19:40,377 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T23:19:40,381 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44261, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T23:19:40,384 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T23:19:40,384 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T23:19:40,393 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T23:19:40,394 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T23:19:40,396 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38841 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:40,399 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=38841 {}] master.ServerManager(486): Registering regionserver=9e5740704f6b,36187,1733440777223 2024-12-05T23:19:40,403 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T23:19:40,403 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T23:19:40,405 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T23:19:40,405 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T23:19:40,412 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T23:19:40,412 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T23:19:40,414 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T23:19:40,416 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/data/hbase/meta/1588230740 2024-12-05T23:19:40,418 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/data/hbase/meta/1588230740 2024-12-05T23:19:40,422 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T23:19:40,427 DEBUG [RS:0;9e5740704f6b:36187 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9 2024-12-05T23:19:40,427 DEBUG [RS:0;9e5740704f6b:36187 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:44761 2024-12-05T23:19:40,427 DEBUG [RS:0;9e5740704f6b:36187 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-12-05T23:19:40,425 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-05T23:19:40,433 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T23:19:40,434 DEBUG [RS:0;9e5740704f6b:36187 {}] zookeeper.ZKUtil(111): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9e5740704f6b,36187,1733440777223 2024-12-05T23:19:40,434 WARN [RS:0;9e5740704f6b:36187 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-05T23:19:40,434 INFO [RS:0;9e5740704f6b:36187 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T23:19:40,434 DEBUG [RS:0;9e5740704f6b:36187 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/WALs/9e5740704f6b,36187,1733440777223 2024-12-05T23:19:40,438 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T23:19:40,440 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64749894, jitterRate=-0.035151392221450806}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T23:19:40,442 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9e5740704f6b,36187,1733440777223] 2024-12-05T23:19:40,446 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-05T23:19:40,446 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-05T23:19:40,447 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-05T23:19:40,447 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-05T23:19:40,447 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T23:19:40,447 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T23:19:40,448 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-05T23:19:40,449 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-05T23:19:40,452 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-12-05T23:19:40,453 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-12-05T23:19:40,463 DEBUG [RS:0;9e5740704f6b:36187 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-12-05T23:19:40,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-05T23:19:40,480 INFO [RS:0;9e5740704f6b:36187 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-05T23:19:40,485 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-05T23:19:40,488 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-05T23:19:40,514 INFO [RS:0;9e5740704f6b:36187 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-05T23:19:40,519 INFO [RS:0;9e5740704f6b:36187 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-05T23:19:40,519 INFO [RS:0;9e5740704f6b:36187 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T23:19:40,521 INFO [RS:0;9e5740704f6b:36187 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-12-05T23:19:40,530 INFO [RS:0;9e5740704f6b:36187 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-05T23:19:40,530 DEBUG [RS:0;9e5740704f6b:36187 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9e5740704f6b:0, corePoolSize=1, maxPoolSize=1 2024-12-05T23:19:40,531 DEBUG [RS:0;9e5740704f6b:36187 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9e5740704f6b:0, corePoolSize=1, maxPoolSize=1 2024-12-05T23:19:40,531 DEBUG [RS:0;9e5740704f6b:36187 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9e5740704f6b:0, corePoolSize=1, maxPoolSize=1 2024-12-05T23:19:40,531 DEBUG [RS:0;9e5740704f6b:36187 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9e5740704f6b:0, corePoolSize=1, maxPoolSize=1 2024-12-05T23:19:40,531 DEBUG [RS:0;9e5740704f6b:36187 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9e5740704f6b:0, corePoolSize=1, maxPoolSize=1 2024-12-05T23:19:40,532 DEBUG [RS:0;9e5740704f6b:36187 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9e5740704f6b:0, corePoolSize=2, maxPoolSize=2 2024-12-05T23:19:40,532 DEBUG [RS:0;9e5740704f6b:36187 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9e5740704f6b:0, corePoolSize=1, maxPoolSize=1 2024-12-05T23:19:40,532 DEBUG [RS:0;9e5740704f6b:36187 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9e5740704f6b:0, corePoolSize=1, maxPoolSize=1 2024-12-05T23:19:40,532 DEBUG [RS:0;9e5740704f6b:36187 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9e5740704f6b:0, corePoolSize=1, maxPoolSize=1 2024-12-05T23:19:40,532 DEBUG [RS:0;9e5740704f6b:36187 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9e5740704f6b:0, corePoolSize=1, maxPoolSize=1 2024-12-05T23:19:40,533 DEBUG [RS:0;9e5740704f6b:36187 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9e5740704f6b:0, corePoolSize=1, maxPoolSize=1 2024-12-05T23:19:40,533 DEBUG [RS:0;9e5740704f6b:36187 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9e5740704f6b:0, corePoolSize=3, maxPoolSize=3 2024-12-05T23:19:40,533 DEBUG [RS:0;9e5740704f6b:36187 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9e5740704f6b:0, corePoolSize=3, maxPoolSize=3 2024-12-05T23:19:40,534 INFO [RS:0;9e5740704f6b:36187 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T23:19:40,534 INFO [RS:0;9e5740704f6b:36187 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-05T23:19:40,534 INFO [RS:0;9e5740704f6b:36187 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-05T23:19:40,534 INFO [RS:0;9e5740704f6b:36187 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-05T23:19:40,534 INFO [RS:0;9e5740704f6b:36187 {}] hbase.ChoreService(168): Chore ScheduledChore name=9e5740704f6b,36187,1733440777223-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T23:19:40,564 INFO [RS:0;9e5740704f6b:36187 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-05T23:19:40,567 INFO [RS:0;9e5740704f6b:36187 {}] hbase.ChoreService(168): Chore ScheduledChore name=9e5740704f6b,36187,1733440777223-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T23:19:40,598 INFO [RS:0;9e5740704f6b:36187 {}] regionserver.Replication(204): 9e5740704f6b,36187,1733440777223 started 2024-12-05T23:19:40,598 INFO [RS:0;9e5740704f6b:36187 {}] regionserver.HRegionServer(1767): Serving as 9e5740704f6b,36187,1733440777223, RpcServer on 9e5740704f6b/172.17.0.2:36187, sessionid=0x10184ed0ca40001 2024-12-05T23:19:40,599 DEBUG [RS:0;9e5740704f6b:36187 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-05T23:19:40,599 DEBUG [RS:0;9e5740704f6b:36187 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:40,599 DEBUG [RS:0;9e5740704f6b:36187 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9e5740704f6b,36187,1733440777223' 2024-12-05T23:19:40,599 DEBUG [RS:0;9e5740704f6b:36187 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-05T23:19:40,600 DEBUG [RS:0;9e5740704f6b:36187 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-05T23:19:40,601 DEBUG [RS:0;9e5740704f6b:36187 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-05T23:19:40,601 DEBUG [RS:0;9e5740704f6b:36187 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-05T23:19:40,602 DEBUG [RS:0;9e5740704f6b:36187 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:40,602 DEBUG [RS:0;9e5740704f6b:36187 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9e5740704f6b,36187,1733440777223' 2024-12-05T23:19:40,602 DEBUG [RS:0;9e5740704f6b:36187 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-05T23:19:40,602 DEBUG [RS:0;9e5740704f6b:36187 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-05T23:19:40,604 DEBUG [RS:0;9e5740704f6b:36187 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-05T23:19:40,604 INFO [RS:0;9e5740704f6b:36187 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-05T23:19:40,604 INFO [RS:0;9e5740704f6b:36187 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-05T23:19:40,639 WARN [9e5740704f6b:38841 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-12-05T23:19:40,711 INFO [RS:0;9e5740704f6b:36187 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-05T23:19:40,714 INFO [RS:0;9e5740704f6b:36187 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9e5740704f6b%2C36187%2C1733440777223, suffix=, logDir=hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/WALs/9e5740704f6b,36187,1733440777223, archiveDir=hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/oldWALs, maxLogs=32 2024-12-05T23:19:40,733 DEBUG [RS:0;9e5740704f6b:36187 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(617): When create output stream for /user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/WALs/9e5740704f6b,36187,1733440777223/9e5740704f6b%2C36187%2C1733440777223.1733440780716, exclude list is [], retry=0 2024-12-05T23:19:40,739 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38377,DS-72ec784f-32f4-4aee-99be-58799187a955,DISK] 2024-12-05T23:19:40,743 INFO [RS:0;9e5740704f6b:36187 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/WALs/9e5740704f6b,36187,1733440777223/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:19:40,744 DEBUG [RS:0;9e5740704f6b:36187 {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41975:41975)] 2024-12-05T23:19:40,891 DEBUG [9e5740704f6b:38841 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-05T23:19:40,895 INFO [PEWorker-4 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9e5740704f6b,36187,1733440777223 2024-12-05T23:19:40,901 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9e5740704f6b,36187,1733440777223, state=OPENING 2024-12-05T23:19:40,908 DEBUG [PEWorker-4 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-05T23:19:40,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T23:19:40,910 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T23:19:40,912 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T23:19:40,912 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T23:19:40,914 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=9e5740704f6b,36187,1733440777223}] 2024-12-05T23:19:41,093 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:41,096 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-05T23:19:41,100 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40406, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-05T23:19:41,111 INFO [RS_OPEN_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-12-05T23:19:41,111 INFO [RS_OPEN_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-05T23:19:41,111 INFO [RS_OPEN_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-05T23:19:41,116 INFO [RS_OPEN_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9e5740704f6b%2C36187%2C1733440777223.meta, suffix=.meta, logDir=hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/WALs/9e5740704f6b,36187,1733440777223, archiveDir=hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/oldWALs, maxLogs=32 2024-12-05T23:19:41,142 DEBUG [RS_OPEN_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(617): When create output stream for /user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/WALs/9e5740704f6b,36187,1733440777223/9e5740704f6b%2C36187%2C1733440777223.meta.1733440781118.meta, exclude list is [], retry=0 2024-12-05T23:19:41,147 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:38377,DS-72ec784f-32f4-4aee-99be-58799187a955,DISK] 2024-12-05T23:19:41,150 INFO [RS_OPEN_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/WALs/9e5740704f6b,36187,1733440777223/9e5740704f6b%2C36187%2C1733440777223.meta.1733440781118.meta 2024-12-05T23:19:41,151 DEBUG [RS_OPEN_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41975:41975)] 2024-12-05T23:19:41,151 DEBUG [RS_OPEN_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-05T23:19:41,153 DEBUG [RS_OPEN_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-05T23:19:41,222 DEBUG [RS_OPEN_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-05T23:19:41,227 INFO [RS_OPEN_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-05T23:19:41,231 DEBUG [RS_OPEN_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-05T23:19:41,231 DEBUG [RS_OPEN_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T23:19:41,232 DEBUG [RS_OPEN_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-12-05T23:19:41,232 DEBUG [RS_OPEN_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-12-05T23:19:41,235 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-05T23:19:41,237 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-05T23:19:41,237 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T23:19:41,238 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T23:19:41,238 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-05T23:19:41,240 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-05T23:19:41,240 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T23:19:41,240 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T23:19:41,241 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-05T23:19:41,242 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-05T23:19:41,242 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T23:19:41,243 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-05T23:19:41,244 DEBUG [RS_OPEN_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/data/hbase/meta/1588230740 2024-12-05T23:19:41,247 DEBUG [RS_OPEN_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/data/hbase/meta/1588230740 2024-12-05T23:19:41,249 DEBUG [RS_OPEN_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-05T23:19:41,252 DEBUG [RS_OPEN_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-05T23:19:41,253 INFO [RS_OPEN_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61493029, jitterRate=-0.08368246257305145}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-05T23:19:41,255 DEBUG [RS_OPEN_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-05T23:19:41,262 INFO [RS_OPEN_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733440781087 2024-12-05T23:19:41,275 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9e5740704f6b,36187,1733440777223 2024-12-05T23:19:41,277 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9e5740704f6b,36187,1733440777223, state=OPEN 2024-12-05T23:19:41,277 DEBUG [RS_OPEN_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-05T23:19:41,278 INFO [RS_OPEN_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-12-05T23:19:41,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T23:19:41,283 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-05T23:19:41,283 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T23:19:41,283 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-05T23:19:41,287 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-12-05T23:19:41,288 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=9e5740704f6b,36187,1733440777223 in 369 msec 2024-12-05T23:19:41,293 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-12-05T23:19:41,293 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 820 msec 2024-12-05T23:19:41,298 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.2790 sec 2024-12-05T23:19:41,298 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733440781298, completionTime=-1 2024-12-05T23:19:41,298 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-05T23:19:41,299 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-12-05T23:19:41,339 DEBUG [hconnection-0x2d26ff7d-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T23:19:41,342 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40416, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T23:19:41,352 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-12-05T23:19:41,352 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733440841352 2024-12-05T23:19:41,352 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733440901352 2024-12-05T23:19:41,352 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 52 msec 2024-12-05T23:19:41,374 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9e5740704f6b,38841,1733440775912-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-05T23:19:41,375 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9e5740704f6b,38841,1733440775912-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T23:19:41,375 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9e5740704f6b,38841,1733440775912-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T23:19:41,376 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9e5740704f6b:38841, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T23:19:41,376 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-05T23:19:41,381 DEBUG [master/9e5740704f6b:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-12-05T23:19:41,385 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-12-05T23:19:41,387 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-05T23:19:41,394 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-12-05T23:19:41,398 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-12-05T23:19:41,399 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T23:19:41,400 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-05T23:19:41,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741835_1011 (size=358) 2024-12-05T23:19:41,815 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 4cc297f79f625a9804bd9a56700799f5, NAME => 'hbase:namespace,,1733440781386.4cc297f79f625a9804bd9a56700799f5.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9 2024-12-05T23:19:41,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741836_1012 (size=42) 2024-12-05T23:19:42,227 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733440781386.4cc297f79f625a9804bd9a56700799f5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T23:19:42,228 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 4cc297f79f625a9804bd9a56700799f5, disabling compactions & flushes 2024-12-05T23:19:42,228 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733440781386.4cc297f79f625a9804bd9a56700799f5. 2024-12-05T23:19:42,228 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733440781386.4cc297f79f625a9804bd9a56700799f5. 2024-12-05T23:19:42,228 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733440781386.4cc297f79f625a9804bd9a56700799f5. after waiting 0 ms 2024-12-05T23:19:42,228 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733440781386.4cc297f79f625a9804bd9a56700799f5. 2024-12-05T23:19:42,228 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733440781386.4cc297f79f625a9804bd9a56700799f5. 2024-12-05T23:19:42,229 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 4cc297f79f625a9804bd9a56700799f5: 2024-12-05T23:19:42,232 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-12-05T23:19:42,240 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733440781386.4cc297f79f625a9804bd9a56700799f5.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733440782233"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733440782233"}]},"ts":"1733440782233"} 2024-12-05T23:19:42,268 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-05T23:19:42,271 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-05T23:19:42,274 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733440782271"}]},"ts":"1733440782271"} 2024-12-05T23:19:42,280 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-12-05T23:19:42,288 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=4cc297f79f625a9804bd9a56700799f5, ASSIGN}] 2024-12-05T23:19:42,291 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=4cc297f79f625a9804bd9a56700799f5, ASSIGN 2024-12-05T23:19:42,293 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=4cc297f79f625a9804bd9a56700799f5, ASSIGN; state=OFFLINE, location=9e5740704f6b,36187,1733440777223; forceNewPlan=false, retain=false 2024-12-05T23:19:42,449 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=4cc297f79f625a9804bd9a56700799f5, regionState=OPENING, regionLocation=9e5740704f6b,36187,1733440777223 2024-12-05T23:19:42,454 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 4cc297f79f625a9804bd9a56700799f5, server=9e5740704f6b,36187,1733440777223}] 2024-12-05T23:19:42,621 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:42,629 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733440781386.4cc297f79f625a9804bd9a56700799f5. 2024-12-05T23:19:42,630 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 4cc297f79f625a9804bd9a56700799f5, NAME => 'hbase:namespace,,1733440781386.4cc297f79f625a9804bd9a56700799f5.', STARTKEY => '', ENDKEY => ''} 2024-12-05T23:19:42,630 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 4cc297f79f625a9804bd9a56700799f5 2024-12-05T23:19:42,630 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733440781386.4cc297f79f625a9804bd9a56700799f5.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-05T23:19:42,631 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 4cc297f79f625a9804bd9a56700799f5 2024-12-05T23:19:42,631 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 4cc297f79f625a9804bd9a56700799f5 2024-12-05T23:19:42,642 INFO [StoreOpener-4cc297f79f625a9804bd9a56700799f5-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 4cc297f79f625a9804bd9a56700799f5 2024-12-05T23:19:42,645 INFO [StoreOpener-4cc297f79f625a9804bd9a56700799f5-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4cc297f79f625a9804bd9a56700799f5 columnFamilyName info 2024-12-05T23:19:42,645 DEBUG [StoreOpener-4cc297f79f625a9804bd9a56700799f5-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-05T23:19:42,646 INFO [StoreOpener-4cc297f79f625a9804bd9a56700799f5-1 {}] regionserver.HStore(327): Store=4cc297f79f625a9804bd9a56700799f5/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-05T23:19:42,649 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/data/hbase/namespace/4cc297f79f625a9804bd9a56700799f5 2024-12-05T23:19:42,650 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/data/hbase/namespace/4cc297f79f625a9804bd9a56700799f5 2024-12-05T23:19:42,655 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 4cc297f79f625a9804bd9a56700799f5 2024-12-05T23:19:42,660 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/data/hbase/namespace/4cc297f79f625a9804bd9a56700799f5/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-05T23:19:42,661 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 4cc297f79f625a9804bd9a56700799f5; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72896394, jitterRate=0.08624091744422913}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-05T23:19:42,664 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 4cc297f79f625a9804bd9a56700799f5: 2024-12-05T23:19:42,668 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733440781386.4cc297f79f625a9804bd9a56700799f5., pid=6, masterSystemTime=1733440782620 2024-12-05T23:19:42,672 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733440781386.4cc297f79f625a9804bd9a56700799f5. 2024-12-05T23:19:42,673 INFO [RS_OPEN_PRIORITY_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733440781386.4cc297f79f625a9804bd9a56700799f5. 2024-12-05T23:19:42,675 INFO [PEWorker-1 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=4cc297f79f625a9804bd9a56700799f5, regionState=OPEN, openSeqNum=2, regionLocation=9e5740704f6b,36187,1733440777223 2024-12-05T23:19:42,693 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-12-05T23:19:42,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 4cc297f79f625a9804bd9a56700799f5, server=9e5740704f6b,36187,1733440777223 in 233 msec 2024-12-05T23:19:42,701 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-12-05T23:19:42,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=4cc297f79f625a9804bd9a56700799f5, ASSIGN in 405 msec 2024-12-05T23:19:42,705 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-05T23:19:42,705 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733440782705"}]},"ts":"1733440782705"} 2024-12-05T23:19:42,709 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-12-05T23:19:42,715 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-12-05T23:19:42,720 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.3280 sec 2024-12-05T23:19:42,805 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-12-05T23:19:42,808 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-12-05T23:19:42,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T23:19:42,809 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T23:19:42,846 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-12-05T23:19:42,866 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-05T23:19:42,874 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 33 msec 2024-12-05T23:19:42,883 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-12-05T23:19:42,900 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-05T23:19:42,908 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 25 msec 2024-12-05T23:19:42,932 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-12-05T23:19:42,939 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-12-05T23:19:42,939 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 5.327sec 2024-12-05T23:19:42,941 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-05T23:19:42,943 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-05T23:19:42,944 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-05T23:19:42,945 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-05T23:19:42,945 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-05T23:19:42,946 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9e5740704f6b,38841,1733440775912-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-05T23:19:42,947 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9e5740704f6b,38841,1733440775912-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-05T23:19:42,959 DEBUG [master/9e5740704f6b:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-12-05T23:19:42,962 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-05T23:19:42,962 INFO [master/9e5740704f6b:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9e5740704f6b,38841,1733440775912-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-05T23:19:43,031 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1ac37e33 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@d296416 2024-12-05T23:19:43,031 WARN [Time-limited test {}] client.ZKConnectionRegistry(90): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-12-05T23:19:43,040 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@509583da, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:19:43,045 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-05T23:19:43,045 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-05T23:19:43,056 DEBUG [hconnection-0x25ca9396-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-05T23:19:43,064 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40418, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-05T23:19:43,075 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=9e5740704f6b,38841,1733440775912 2024-12-05T23:19:43,077 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1e2eb297 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@49d3634b 2024-12-05T23:19:43,082 DEBUG [ReadOnlyZKClient-127.0.0.1:50903@0x1e2eb297 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@266c5f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:19:43,101 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[0] Thread=221, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=628, ProcessCount=11, AvailableMemoryMB=3060 2024-12-05T23:19:43,161 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching master stub from registry 2024-12-05T23:19:43,162 DEBUG [ReadOnlyZKClient-127.0.0.1:50903@0x1e2eb297 {}] client.AsyncConnectionImpl(310): The fetched master address is 9e5740704f6b,38841,1733440775912 2024-12-05T23:19:43,169 DEBUG [ReadOnlyZKClient-127.0.0.1:50903@0x1e2eb297 {}] client.ConnectionUtils(581): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3916bad0 2024-12-05T23:19:43,171 DEBUG [ReadOnlyZKClient-127.0.0.1:50903@0x1e2eb297 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-05T23:19:43,173 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33460, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-05T23:19:43,175 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:19:43,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:19:43,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-05T23:19:43,210 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:43,212 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:43,215 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:43,215 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40004 connected 2024-12-05T23:19:43,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-05T23:19:43,318 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40004, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:43,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T23:19:43,330 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T23:19:43,341 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:19:43,506 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:43,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-05T23:19:43,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=10 2024-12-05T23:19:43,508 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-05T23:19:43,527 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=10}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:43,549 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:19:43,567 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-05T23:19:43,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=10 2024-12-05T23:19:43,572 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:19:43,575 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-12-05T23:19:43,575 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 232 msec 2024-12-05T23:19:43,576 INFO [PEWorker-2 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-05T23:19:43,578 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 400 msec 2024-12-05T23:19:43,603 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6073bb64 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7e993979 2024-12-05T23:19:43,607 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:19:43,609 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:19:43,610 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@42eefa4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:19:43,612 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:43,615 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:43,616 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:19:43,616 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40006 connected 2024-12-05T23:19:43,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-05T23:19:43,818 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:19:43,821 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-05T23:19:43,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=11, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:19:43,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=11 2024-12-05T23:19:43,854 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:43,855 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:43,859 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:43,860 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40007 connected 2024-12-05T23:19:43,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=11 2024-12-05T23:19:43,962 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40007, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:43,969 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:19:44,122 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:44,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=12 2024-12-05T23:19:44,123 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:19:44,125 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:19:44,125 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:19:44,126 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:19:44,126 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40006, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:44,126 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40006, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:44,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=11 2024-12-05T23:19:44,229 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40006, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:44,229 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=12}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6073bb64 to 127.0.0.1:50903 2024-12-05T23:19:44,230 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=12}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:19:44,247 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:19:44,248 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:19:44,248 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-05T23:19:44,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=12 2024-12-05T23:19:44,252 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:19:44,256 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=12, resume processing ppid=11 2024-12-05T23:19:44,256 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, ppid=11, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 283 msec 2024-12-05T23:19:44,256 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-05T23:19:44,259 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 435 msec 2024-12-05T23:19:44,282 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x758eaba5 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2e88b479 2024-12-05T23:19:44,288 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:19:44,288 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:19:44,289 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@40008926, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:19:44,290 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:44,293 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:44,293 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40009 connected 2024-12-05T23:19:44,293 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:19:44,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=11 2024-12-05T23:19:44,457 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:19:44,461 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:19:44,463 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:19:44,464 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:19:44,467 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-05T23:19:44,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=13, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:19:44,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=13 2024-12-05T23:19:44,486 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:44,487 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:44,490 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:44,490 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4000a connected 2024-12-05T23:19:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=13 2024-12-05T23:19:44,592 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4000a, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:44,601 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:19:44,754 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:44,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=14 2024-12-05T23:19:44,755 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:19:44,756 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:19:44,757 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:19:44,757 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:19:44,757 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40009, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:44,757 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40009, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:44,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=13 2024-12-05T23:19:44,860 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40009, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:44,860 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=14}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x758eaba5 to 127.0.0.1:50903 2024-12-05T23:19:44,860 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=14}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:19:44,879 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:19:44,880 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:19:44,880 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-05T23:19:44,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=14 2024-12-05T23:19:44,885 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:19:44,888 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=14, resume processing ppid=13 2024-12-05T23:19:44,889 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, ppid=13, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 284 msec 2024-12-05T23:19:44,889 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-05T23:19:44,891 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 422 msec 2024-12-05T23:19:44,913 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3ba7d31d to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@331d45db 2024-12-05T23:19:44,917 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:19:44,918 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:19:44,918 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@1707074a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:19:44,919 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:44,924 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:44,924 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:19:44,924 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca4000c connected 2024-12-05T23:19:45,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=13 2024-12-05T23:19:45,097 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:19:45,097 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:19:45,099 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:19:45,100 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:19:45,101 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:19:45,102 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-05T23:19:45,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=15, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:19:45,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-12-05T23:19:45,121 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:45,122 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:45,125 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:45,125 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4000d connected 2024-12-05T23:19:45,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-12-05T23:19:45,227 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4000d, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:45,232 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:19:45,384 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:45,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=16 2024-12-05T23:19:45,386 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:19:45,387 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:19:45,387 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:19:45,387 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:19:45,387 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca4000c, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:45,387 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca4000c, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:45,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-12-05T23:19:45,496 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca4000c, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:45,497 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=16}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3ba7d31d to 127.0.0.1:50903 2024-12-05T23:19:45,497 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=16}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:19:45,512 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:19:45,513 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:19:45,513 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-05T23:19:45,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=16 2024-12-05T23:19:45,517 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:19:45,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=16, resume processing ppid=15 2024-12-05T23:19:45,520 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, ppid=15, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 285 msec 2024-12-05T23:19:45,520 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-05T23:19:45,523 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 418 msec 2024-12-05T23:19:45,543 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x20a4180e to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3086323e 2024-12-05T23:19:45,547 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:19:45,547 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:19:45,547 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@173401de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:19:45,548 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:45,551 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:45,552 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:19:45,552 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca4000f connected 2024-12-05T23:19:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-12-05T23:19:45,737 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:19:45,737 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:19:45,738 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:19:45,739 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:19:45,740 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-05T23:19:45,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=17, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:19:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=17 2024-12-05T23:19:45,757 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:45,758 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:45,762 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:45,763 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40010 connected 2024-12-05T23:19:45,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=17 2024-12-05T23:19:45,865 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40010, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:45,876 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:19:46,028 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:46,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=18 2024-12-05T23:19:46,029 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:19:46,030 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:19:46,031 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:19:46,031 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:19:46,031 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca4000f, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:46,031 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca4000f, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:46,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=17 2024-12-05T23:19:46,134 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca4000f, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:46,134 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=18}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x20a4180e to 127.0.0.1:50903 2024-12-05T23:19:46,134 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=18}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:19:46,149 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:19:46,150 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:19:46,150 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-05T23:19:46,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=18 2024-12-05T23:19:46,153 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:19:46,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=18, resume processing ppid=17 2024-12-05T23:19:46,156 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-05T23:19:46,156 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, ppid=17, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 278 msec 2024-12-05T23:19:46,159 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 416 msec 2024-12-05T23:19:46,179 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x083f5c03 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3da5ee86 2024-12-05T23:19:46,184 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:19:46,185 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:19:46,185 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@2eaed95d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:19:46,186 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:46,189 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:46,189 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:19:46,189 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40012 connected 2024-12-05T23:19:46,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=17 2024-12-05T23:19:46,366 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:19:46,367 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:19:46,368 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:19:46,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-05T23:19:46,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=19, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:19:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=19 2024-12-05T23:19:46,387 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:46,387 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:46,390 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:46,390 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40013 connected 2024-12-05T23:19:46,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=19 2024-12-05T23:19:46,493 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40013, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:46,497 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:19:46,650 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:46,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=20 2024-12-05T23:19:46,651 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:19:46,652 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:19:46,652 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:19:46,652 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:19:46,652 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40012, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:46,653 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40012, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:46,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=19 2024-12-05T23:19:46,740 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T23:19:46,755 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40012, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:46,755 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=20}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x083f5c03 to 127.0.0.1:50903 2024-12-05T23:19:46,755 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=20}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:19:46,770 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:19:46,770 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:19:46,770 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-05T23:19:46,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=20 2024-12-05T23:19:46,774 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:19:46,778 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=20, resume processing ppid=19 2024-12-05T23:19:46,778 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=20, ppid=19, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 277 msec 2024-12-05T23:19:46,778 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-05T23:19:46,780 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=19, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 409 msec 2024-12-05T23:19:46,789 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-05T23:19:46,790 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-12-05T23:19:46,791 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-05T23:19:46,792 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-05T23:19:46,794 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-12-05T23:19:46,794 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2024-12-05T23:19:46,795 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T23:19:46,795 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-05T23:19:46,795 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-05T23:19:46,796 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-05T23:19:46,812 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5148500c to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@21ca46b1 2024-12-05T23:19:46,818 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:19:46,818 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:19:46,818 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3cf685f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:19:46,819 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:46,822 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:46,822 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:19:46,822 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40015 connected 2024-12-05T23:19:46,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=19 2024-12-05T23:19:46,997 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:19:46,997 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:19:46,998 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-05T23:19:46,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=21, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:19:47,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-12-05T23:19:47,016 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:47,017 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:47,019 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:47,019 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40016 connected 2024-12-05T23:19:47,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-12-05T23:19:47,122 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40016, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:47,129 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:19:47,282 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:47,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=22 2024-12-05T23:19:47,283 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:19:47,285 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:19:47,285 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:19:47,285 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:19:47,285 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40015, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:47,285 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40015, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:47,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-12-05T23:19:47,387 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40015, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:47,387 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=22}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5148500c to 127.0.0.1:50903 2024-12-05T23:19:47,387 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=22}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:19:47,402 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:19:47,402 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:19:47,403 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-05T23:19:47,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=22 2024-12-05T23:19:47,406 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:19:47,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=22, resume processing ppid=21 2024-12-05T23:19:47,409 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=22, ppid=21, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 277 msec 2024-12-05T23:19:47,409 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-05T23:19:47,411 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=21, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 411 msec 2024-12-05T23:19:47,435 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x228474d2 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@768178a7 2024-12-05T23:19:47,439 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:19:47,440 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:19:47,440 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@63b6ad87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:19:47,441 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:47,447 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:47,447 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:19:47,447 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40018 connected 2024-12-05T23:19:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-12-05T23:19:47,626 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:19:47,627 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:19:47,630 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:19:47,630 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:19:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=24, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:19:47,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=23, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:19:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-12-05T23:19:47,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-12-05T23:19:47,636 DEBUG [PEWorker-1 {}] procedure.MasterProcedureScheduler(583): PEER '1', shared lock count=0, exclusively locked by procId=24 2024-12-05T23:19:47,636 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1470): LOCK_EVENT_WAIT pid=23, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:19:47,637 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-05T23:19:47,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-05T23:19:47,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:19:47,639 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:19:47,641 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:19:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-12-05T23:19:47,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-12-05T23:19:47,793 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:47,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=25 2024-12-05T23:19:47,794 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-05T23:19:47,794 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-05T23:19:47,794 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-05T23:19:47,794 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:19:47,794 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40018, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:47,795 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40018, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:47,898 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40018, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:47,898 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=25}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x228474d2 to 127.0.0.1:50903 2024-12-05T23:19:47,898 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=25}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:19:47,898 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-05T23:19:47,900 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=25}] zookeeper.ZKUtil(111): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:19:47,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:19:47,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:19:47,904 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:19:47,905 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=25}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:19:47,905 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=25 2024-12-05T23:19:47,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=25 2024-12-05T23:19:47,908 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:19:47,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=25, resume processing ppid=24 2024-12-05T23:19:47,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=25, ppid=24, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 267 msec 2024-12-05T23:19:47,915 DEBUG [PEWorker-2 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:19:47,915 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-05T23:19:47,917 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=24, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 285 msec 2024-12-05T23:19:47,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-12-05T23:19:47,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-12-05T23:19:47,957 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:19:47,957 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:19:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=26, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:19:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-12-05T23:19:47,960 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:19:47,966 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=26, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:19:47,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=26, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=9 msec 2024-12-05T23:19:47,967 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:19:48,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-12-05T23:19:48,068 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:19:48,070 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=testing utility connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:48,088 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility0x0, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:48,102 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): testing utility-0x10184ed0ca40019 connected 2024-12-05T23:19:48,135 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[0] Thread=222 (was 221) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=452 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=602 (was 628), ProcessCount=11 (was 11), AvailableMemoryMB=2993 (was 3060) 2024-12-05T23:19:48,163 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[0] Thread=222, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=602, ProcessCount=11, AvailableMemoryMB=2990 2024-12-05T23:19:48,166 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:19:48,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=27, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:19:48,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-12-05T23:19:48,202 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:48,204 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:48,214 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:48,224 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4001a connected 2024-12-05T23:19:48,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-12-05T23:19:48,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-12-05T23:19:48,327 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4001a, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:48,340 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:19:48,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-12-05T23:19:48,510 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:48,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=28 2024-12-05T23:19:48,512 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-05T23:19:48,541 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=28}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:48,564 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:19:48,592 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-12-05T23:19:48,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=28 2024-12-05T23:19:48,598 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:19:48,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=28, resume processing ppid=27 2024-12-05T23:19:48,601 INFO [PEWorker-2 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-05T23:19:48,601 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=28, ppid=27, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 258 msec 2024-12-05T23:19:48,603 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=27, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 435 msec 2024-12-05T23:19:48,638 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x777f35ab to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@24382163 2024-12-05T23:19:48,669 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:19:48,670 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:19:48,670 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3993ba99, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:19:48,672 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:48,685 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:48,686 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:19:48,686 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca4001c connected 2024-12-05T23:19:48,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-12-05T23:19:48,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-12-05T23:19:48,797 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:19:48,798 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-05T23:19:48,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=29, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:19:48,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=29 2024-12-05T23:19:48,819 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:48,820 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:48,840 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:48,844 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4001d connected 2024-12-05T23:19:48,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=29 2024-12-05T23:19:48,947 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4001d, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:48,955 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:19:49,113 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:49,114 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=30 2024-12-05T23:19:49,114 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:19:49,116 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:19:49,116 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:19:49,116 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:19:49,116 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca4001c, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:49,117 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca4001c, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:49,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=29 2024-12-05T23:19:49,220 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca4001c, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:49,221 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=30}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x777f35ab to 127.0.0.1:50903 2024-12-05T23:19:49,221 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=30}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:19:49,245 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:19:49,246 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:19:49,248 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-12-05T23:19:49,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=30 2024-12-05T23:19:49,255 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:19:49,261 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=30, resume processing ppid=29 2024-12-05T23:19:49,261 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=30, ppid=29, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 300 msec 2024-12-05T23:19:49,262 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-05T23:19:49,264 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=29, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 464 msec 2024-12-05T23:19:49,312 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0b0e0c53 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@26ca9a3a 2024-12-05T23:19:49,344 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:19:49,344 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:19:49,344 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@1aca1af3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:19:49,349 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:49,357 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:19:49,360 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:49,363 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca4001f connected 2024-12-05T23:19:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=29 2024-12-05T23:19:49,426 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:19:49,427 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:19:49,429 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-05T23:19:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=31, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:19:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=31 2024-12-05T23:19:49,453 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:49,454 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:49,459 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:49,460 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40020 connected 2024-12-05T23:19:49,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=31 2024-12-05T23:19:49,561 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40020, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:49,570 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:19:49,727 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:49,728 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=32 2024-12-05T23:19:49,728 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:19:49,730 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:19:49,730 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:19:49,730 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:19:49,730 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca4001f, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:49,731 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca4001f, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:49,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=31 2024-12-05T23:19:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-12-05T23:19:49,835 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca4001f, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:49,835 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=32}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0b0e0c53 to 127.0.0.1:50903 2024-12-05T23:19:49,835 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=32}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:19:49,865 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:19:49,865 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:19:49,866 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=32 2024-12-05T23:19:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=32 2024-12-05T23:19:49,871 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:19:49,874 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=32, resume processing ppid=31 2024-12-05T23:19:49,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=32, ppid=31, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 301 msec 2024-12-05T23:19:49,875 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-05T23:19:49,877 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=31, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 446 msec 2024-12-05T23:19:49,927 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0f21ace1 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5c2894c2 2024-12-05T23:19:49,938 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:19:49,938 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:19:49,939 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@4f3f18b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:19:49,941 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:49,946 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:49,946 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:19:49,964 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40022 connected 2024-12-05T23:19:50,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=31 2024-12-05T23:19:50,057 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:19:50,058 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:19:50,060 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns1:testNamespacesAndTableCfsConfigConflict1=null},bandwidth=0,serial=false 2024-12-05T23:19:50,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=33, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:19:50,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-12-05T23:19:50,087 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:50,088 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:50,095 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:50,095 ERROR [PEWorker-3-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$360/0x00007f9a60914e50@605cdf41 rejected from java.util.concurrent.ThreadPoolExecutor@255e4d2e[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 0] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.PendingWatcher.process(PendingWatcher.java:38) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-12-05T23:19:50,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-12-05T23:19:50,198 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:50,198 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:19:50,201 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=33, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:19:50,204 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=33, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=141 msec 2024-12-05T23:19:50,204 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:19:50,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-12-05T23:19:50,378 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config 2024-12-05T23:19:50,379 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:19:50,380 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-05T23:19:50,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=34, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:19:50,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-12-05T23:19:50,409 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:50,411 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:50,417 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:50,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-12-05T23:19:50,520 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:50,530 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:19:50,683 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:50,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=35 2024-12-05T23:19:50,685 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:19:50,689 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:19:50,689 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:19:50,689 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:19:50,690 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40022, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:50,690 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40022, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:50,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-12-05T23:19:50,793 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40022, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:50,793 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=35}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0f21ace1 to 127.0.0.1:50903 2024-12-05T23:19:50,793 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=35}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:19:50,816 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:19:50,817 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:19:50,817 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-12-05T23:19:50,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=35 2024-12-05T23:19:50,821 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:19:50,825 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=35, resume processing ppid=34 2024-12-05T23:19:50,825 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=35, ppid=34, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 292 msec 2024-12-05T23:19:50,825 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-05T23:19:50,828 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 445 msec 2024-12-05T23:19:50,868 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x69f2cf55 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@dd3377f 2024-12-05T23:19:50,897 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:19:50,898 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:19:50,898 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@63cbe0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:19:50,901 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:50,922 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:50,923 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:19:50,925 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40026 connected 2024-12-05T23:19:51,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-12-05T23:19:51,017 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:19:51,018 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:19:51,019 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-05T23:19:51,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=36, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:19:51,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-12-05T23:19:51,048 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:51,049 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:51,055 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:51,055 ERROR [PEWorker-3-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$360/0x00007f9a60914e50@23834ad3 rejected from java.util.concurrent.ThreadPoolExecutor@27bcca7b[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 0] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.PendingWatcher.process(PendingWatcher.java:38) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-12-05T23:19:51,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-12-05T23:19:51,161 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:51,161 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:19:51,164 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=36, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:19:51,166 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=36, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=145 msec 2024-12-05T23:19:51,166 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:19:51,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-12-05T23:19:51,348 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config 2024-12-05T23:19:51,350 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:19:51,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=37, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:19:51,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=37 2024-12-05T23:19:51,357 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-05T23:19:51,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-05T23:19:51,360 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:19:51,361 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:19:51,362 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:19:51,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=37 2024-12-05T23:19:51,515 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:51,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=38 2024-12-05T23:19:51,517 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-05T23:19:51,517 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-05T23:19:51,517 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-05T23:19:51,517 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:19:51,517 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40026, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:51,518 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40026, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:51,623 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40026, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:51,623 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=38}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x69f2cf55 to 127.0.0.1:50903 2024-12-05T23:19:51,623 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=38}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:19:51,623 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-05T23:19:51,626 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=38}] zookeeper.ZKUtil(111): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:19:51,629 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:19:51,630 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:19:51,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:19:51,635 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=38}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:19:51,636 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=38 2024-12-05T23:19:51,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=38 2024-12-05T23:19:51,640 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:19:51,644 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=38, resume processing ppid=37 2024-12-05T23:19:51,644 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=38, ppid=37, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 278 msec 2024-12-05T23:19:51,648 DEBUG [PEWorker-1 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:19:51,648 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-05T23:19:51,651 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=37, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 298 msec 2024-12-05T23:19:51,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=37 2024-12-05T23:19:51,678 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:19:51,680 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:19:51,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=39, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:19:51,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-12-05T23:19:51,683 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:19:51,685 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=39, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:19:51,686 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=39, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-05T23:19:51,686 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:19:51,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-12-05T23:19:51,788 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-05T23:19:51,789 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:19:51,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=40, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:19:51,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=40 2024-12-05T23:19:51,796 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:19:51,800 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=40, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:19:51,802 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=40, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=12 msec 2024-12-05T23:19:51,803 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:19:51,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-12-05T23:19:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=40 2024-12-05T23:19:51,897 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:19:51,927 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[0] Thread=222 (was 222), OpenFileDescriptor=450 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=618 (was 602) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2944 (was 2990) 2024-12-05T23:19:51,944 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAddRemovePeer[0] Thread=222, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=618, ProcessCount=11, AvailableMemoryMB=2944 2024-12-05T23:19:51,947 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:19:51,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=41, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:19:51,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41 2024-12-05T23:19:51,987 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:51,988 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:52,004 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:52,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41 2024-12-05T23:19:52,106 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:52,114 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:19:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41 2024-12-05T23:19:52,267 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:52,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=42 2024-12-05T23:19:52,268 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=42}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-05T23:19:52,288 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=42}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:52,305 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=42}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:19:52,319 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=42}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=42 2024-12-05T23:19:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=42 2024-12-05T23:19:52,323 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:19:52,328 INFO [PEWorker-3 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-05T23:19:52,328 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=42, resume processing ppid=41 2024-12-05T23:19:52,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=42, ppid=41, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 209 msec 2024-12-05T23:19:52,331 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=41, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 381 msec 2024-12-05T23:19:52,372 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x46001c4e to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6227b5d4 2024-12-05T23:19:52,390 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:19:52,391 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:19:52,392 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@47604b9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:19:52,398 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:52,408 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:19:52,408 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:52,412 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca4002a connected 2024-12-05T23:19:52,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41 2024-12-05T23:19:52,577 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:19:52,578 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:19:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=43, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:19:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=43 2024-12-05T23:19:52,616 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:52,617 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:52,639 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:52,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=43 2024-12-05T23:19:52,742 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:52,742 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:121) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:19:52,745 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=43, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:121) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:19:52,748 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=43, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=167 msec 2024-12-05T23:19:52,748 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:19:52,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=43 2024-12-05T23:19:52,908 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Replication peer 1 already exists 2024-12-05T23:19:52,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-05T23:19:52,918 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:19:52,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=44, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:19:52,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=44 2024-12-05T23:19:52,923 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:19:52,925 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=44, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:19:52,926 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=44, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec 2024-12-05T23:19:52,926 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:19:53,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=44 2024-12-05T23:19:53,028 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:19:53,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-05T23:19:53,030 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=2, config=clusterKey=127.0.0.1:50903:/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:19:53,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=45, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:19:53,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-12-05T23:19:53,052 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:53,052 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:53,055 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:53,056 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4002c connected 2024-12-05T23:19:53,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-12-05T23:19:53,158 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4002c, quorum=127.0.0.1:50903, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:53,170 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:19:53,322 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:53,323 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=46 2024-12-05T23:19:53,323 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=2, type=ADD_PEER 2024-12-05T23:19:53,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-12-05T23:19:53,373 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=46}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:53,403 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T23:19:53,409 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 2000 millis 2024-12-05T23:19:53,438 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.ReplicationSource(229): queueId=2, ReplicationSource: 2, currentBandwidth=0 2024-12-05T23:19:53,456 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=46 2024-12-05T23:19:53,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=46 2024-12-05T23:19:53,461 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 2 for ADD on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:19:53,464 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=46, resume processing ppid=45 2024-12-05T23:19:53,465 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=46, ppid=45, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 293 msec 2024-12-05T23:19:53,465 INFO [PEWorker-2 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 2, config clusterKey=127.0.0.1:50903:/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-05T23:19:53,467 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=45, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 435 msec 2024-12-05T23:19:53,512 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x25d5057e to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3c5ed9e0 2024-12-05T23:19:53,518 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,2 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:19:53,518 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,2 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:19:53,519 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@464e0629, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:19:53,521 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 2 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:53,526 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,2-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 20x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:53,527 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,2 {}] regionserver.ReplicationSource(569): peerId=2, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:19:53,527 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 2-0x10184ed0ca4002e connected 2024-12-05T23:19:53,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-12-05T23:19:53,656 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 2 completed 2024-12-05T23:19:53,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-05T23:19:53,658 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:19:53,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=47, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:19:53,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-12-05T23:19:53,663 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-05T23:19:53,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-05T23:19:53,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:19:53,666 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:19:53,667 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:19:53,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-12-05T23:19:53,820 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:53,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=48 2024-12-05T23:19:53,821 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-05T23:19:53,822 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-05T23:19:53,822 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-05T23:19:53,822 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:19:53,822 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca4002a, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:53,822 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca4002a, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:53,926 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca4002a, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:53,926 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=48}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x46001c4e to 127.0.0.1:50903 2024-12-05T23:19:53,926 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=48}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:19:53,927 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-05T23:19:53,929 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=48}] zookeeper.ZKUtil(111): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:19:53,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:19:53,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:19:53,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:19:53,935 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=48}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:19:53,936 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=48 2024-12-05T23:19:53,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=48 2024-12-05T23:19:53,940 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:19:53,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=48, resume processing ppid=47 2024-12-05T23:19:53,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=48, ppid=47, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 273 msec 2024-12-05T23:19:53,957 DEBUG [PEWorker-1 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:19:53,957 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-05T23:19:53,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=47, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 299 msec 2024-12-05T23:19:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-12-05T23:19:53,976 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:19:53,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-05T23:19:53,978 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:19:53,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=49, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:19:53,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-12-05T23:19:53,983 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/peer-state 2024-12-05T23:19:53,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/peer-state 2024-12-05T23:19:53,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/2 2024-12-05T23:19:53,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2 2024-12-05T23:19:53,989 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:19:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-12-05T23:19:54,142 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:54,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=50 2024-12-05T23:19:54,144 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=2, type=REMOVE_PEER 2024-12-05T23:19:54,144 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 2: 0 2024-12-05T23:19:54,144 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.ReplicationSource(657): peerId=2, Closing source 2 because: Replication stream was removed by a user 2024-12-05T23:19:54,144 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,2 {}] regionserver.ReplicationSource(508): peerId=2, Interrupted while sleeping between retries 2024-12-05T23:19:54,144 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,2 {}] zookeeper.ZKUtil(165): connection to cluster: 2-0x10184ed0ca4002e, quorum=127.0.0.1:50903, baseZNode=/hbase-test2 Unable to set watcher on znode (/hbase-test2/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:54,145 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,2 {}] zookeeper.ZKWatcher(734): connection to cluster: 2-0x10184ed0ca4002e, quorum=127.0.0.1:50903, baseZNode=/hbase-test2 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:54,247 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=50}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x25d5057e to 127.0.0.1:50903 2024-12-05T23:19:54,248 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=50}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:19:54,248 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.ReplicationSourceManager(430): Done with the queue 2 2024-12-05T23:19:54,249 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,2-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 2-0x10184ed0ca4002e, quorum=127.0.0.1:50903, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:54,253 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=50}] zookeeper.ZKUtil(111): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/2/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:19:54,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/2/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:19:54,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/2 2024-12-05T23:19:54,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/2 2024-12-05T23:19:54,260 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=50}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/2 not found in hfile reference queue. 2024-12-05T23:19:54,260 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=50 2024-12-05T23:19:54,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=50 2024-12-05T23:19:54,264 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 2 for REMOVE on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:19:54,269 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=50, resume processing ppid=49 2024-12-05T23:19:54,269 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=50, ppid=49, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 275 msec 2024-12-05T23:19:54,272 DEBUG [PEWorker-3 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/2 not found in hfile reference queue. 2024-12-05T23:19:54,272 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(76): Successfully removed peer 2 2024-12-05T23:19:54,274 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=49, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 294 msec 2024-12-05T23:19:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-12-05T23:19:54,297 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 2 completed 2024-12-05T23:19:54,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-05T23:19:54,298 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:19:54,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=51, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:19:54,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-12-05T23:19:54,300 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:19:54,302 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=51, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:19:54,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=51, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-05T23:19:54,303 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:19:54,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-12-05T23:19:54,407 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-05T23:19:54,408 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:19:54,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=52, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:19:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=52 2024-12-05T23:19:54,411 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:19:54,412 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=52, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:19:54,413 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=52, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-05T23:19:54,414 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:19:54,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=52 2024-12-05T23:19:54,518 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:19:54,540 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAddRemovePeer[0] Thread=219 (was 222), OpenFileDescriptor=446 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=618 (was 618), ProcessCount=11 (was 11), AvailableMemoryMB=2912 (was 2944) 2024-12-05T23:19:54,561 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[0] Thread=219, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=618, ProcessCount=11, AvailableMemoryMB=2912 2024-12-05T23:19:54,562 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=whatever,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:19:54,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=53, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:19:54,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53 2024-12-05T23:19:54,574 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:291) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:288) ~[classes/:?] ... 12 more 2024-12-05T23:19:54,577 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=53, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:291) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:288) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:19:54,600 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=53, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=15 msec 2024-12-05T23:19:54,600 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:19:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53 2024-12-05T23:19:54,678 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Can not instantiate configured replication endpoint class=whatever 2024-12-05T23:19:54,696 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:19:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=54, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:19:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-12-05T23:19:54,713 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:19:54,715 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=54, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:19:54,716 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=54, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=19 msec 2024-12-05T23:19:54,716 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:19:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-12-05T23:19:54,818 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-05T23:19:54,818 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:19:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=55, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:19:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-12-05T23:19:54,828 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:19:54,832 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=55, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:19:54,837 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=55, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=15 msec 2024-12-05T23:19:54,838 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:19:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-12-05T23:19:54,927 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:19:54,958 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[0] Thread=219 (was 219), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=618 (was 618), ProcessCount=11 (was 11), AvailableMemoryMB=2907 (was 2912) 2024-12-05T23:19:54,970 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[0] Thread=219, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=618, ProcessCount=11, AvailableMemoryMB=2907 2024-12-05T23:19:54,974 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=whatever,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:19:54,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=56, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:19:54,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=56 2024-12-05T23:19:54,985 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] ... 13 more 2024-12-05T23:19:54,987 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=56, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:19:54,989 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=56, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=14 msec 2024-12-05T23:19:54,989 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:19:55,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=56 2024-12-05T23:19:55,088 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Invalid cluster key: whatever 2024-12-05T23:19:55,089 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:19:55,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=57, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:19:55,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-12-05T23:19:55,093 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:19:55,095 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=57, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:19:55,097 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=57, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=7 msec 2024-12-05T23:19:55,097 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:19:55,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-12-05T23:19:55,198 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-05T23:19:55,198 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:19:55,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=58, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:19:55,202 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:19:55,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=58 2024-12-05T23:19:55,203 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=58, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:19:55,205 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=58, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-05T23:19:55,205 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:19:55,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=58 2024-12-05T23:19:55,307 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:19:55,329 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[0] Thread=219 (was 219), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=618 (was 618), ProcessCount=11 (was 11), AvailableMemoryMB=2904 (was 2907) 2024-12-05T23:19:55,357 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[0] Thread=219, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=618, ProcessCount=11, AvailableMemoryMB=2903 2024-12-05T23:19:55,358 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:19:55,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=59, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:19:55,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-12-05T23:19:55,390 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:55,392 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:55,412 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:55,416 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4002f connected 2024-12-05T23:19:55,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-12-05T23:19:55,519 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4002f, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:55,531 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:19:55,686 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:55,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-12-05T23:19:55,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=60 2024-12-05T23:19:55,687 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-05T23:19:55,715 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=60}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:55,760 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:19:55,780 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=60 2024-12-05T23:19:55,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=60 2024-12-05T23:19:55,790 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:19:55,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=60, resume processing ppid=59 2024-12-05T23:19:55,795 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=60, ppid=59, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 259 msec 2024-12-05T23:19:55,795 INFO [PEWorker-1 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-05T23:19:55,798 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=59, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 437 msec 2024-12-05T23:19:55,836 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x22bbb6dd to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@689df427 2024-12-05T23:19:55,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-12-05T23:19:55,852 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:19:55,853 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:19:55,853 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3274012c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:19:55,854 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:55,867 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:55,868 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:19:55,870 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40031 connected 2024-12-05T23:19:55,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-12-05T23:19:55,998 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:19:55,999 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-05T23:19:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=61, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:19:56,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-12-05T23:19:56,043 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:56,044 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:56,061 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:56,061 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40032 connected 2024-12-05T23:19:56,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-12-05T23:19:56,171 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40032, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:56,181 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:19:56,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-12-05T23:19:56,334 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:56,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=62 2024-12-05T23:19:56,335 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:19:56,337 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:19:56,337 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:19:56,337 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:19:56,338 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40031, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:56,338 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40031, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:56,440 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40031, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:56,440 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=62}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x22bbb6dd to 127.0.0.1:50903 2024-12-05T23:19:56,440 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=62}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:19:56,457 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:19:56,457 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:19:56,458 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-12-05T23:19:56,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=62 2024-12-05T23:19:56,465 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:19:56,468 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=62, resume processing ppid=61 2024-12-05T23:19:56,468 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=62, ppid=61, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 285 msec 2024-12-05T23:19:56,468 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-05T23:19:56,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=61, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 470 msec 2024-12-05T23:19:56,492 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7f34978c to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7a4d2f0c 2024-12-05T23:19:56,496 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:19:56,497 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:19:56,497 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@7cdc4808, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:19:56,498 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:56,508 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:56,508 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:19:56,508 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40034 connected 2024-12-05T23:19:56,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-12-05T23:19:56,627 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:19:56,628 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:19:56,629 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-12-05T23:19:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=63, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:19:56,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-12-05T23:19:56,655 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:56,656 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:56,659 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:56,659 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40035 connected 2024-12-05T23:19:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-12-05T23:19:56,761 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40035, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:56,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:19:56,919 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:56,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=64 2024-12-05T23:19:56,920 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:19:56,922 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:19:56,922 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:19:56,923 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:19:56,923 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40034, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:56,923 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40034, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-12-05T23:19:57,047 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40034, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:57,047 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=64}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7f34978c to 127.0.0.1:50903 2024-12-05T23:19:57,048 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=64}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:19:57,070 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:19:57,071 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:19:57,071 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=64 2024-12-05T23:19:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=64 2024-12-05T23:19:57,074 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:19:57,078 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=64, resume processing ppid=63 2024-12-05T23:19:57,078 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=64, ppid=63, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 308 msec 2024-12-05T23:19:57,078 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-12-05T23:19:57,080 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=63, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 449 msec 2024-12-05T23:19:57,119 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6f008005 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@30ab0642 2024-12-05T23:19:57,137 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:19:57,138 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:19:57,138 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@4a6c6e74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:19:57,139 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:57,143 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:57,143 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:19:57,155 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40037 connected 2024-12-05T23:19:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-12-05T23:19:57,257 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:19:57,258 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:19:57,259 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:19:57,260 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-05T23:19:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=65, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:19:57,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-12-05T23:19:57,292 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:57,293 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:57,305 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-12-05T23:19:57,406 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:57,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:19:57,563 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:57,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=66 2024-12-05T23:19:57,564 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:19:57,565 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:19:57,566 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:19:57,566 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:19:57,566 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40037, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:57,566 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40037, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:57,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-12-05T23:19:57,668 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40037, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:57,669 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=66}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6f008005 to 127.0.0.1:50903 2024-12-05T23:19:57,669 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=66}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:19:57,699 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:19:57,699 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:19:57,700 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-12-05T23:19:57,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=66 2024-12-05T23:19:57,706 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:19:57,710 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=66, resume processing ppid=65 2024-12-05T23:19:57,710 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=66, ppid=65, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 295 msec 2024-12-05T23:19:57,710 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-05T23:19:57,712 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=65, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 450 msec 2024-12-05T23:19:57,744 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x75abc9b3 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6ee3b3d1 2024-12-05T23:19:57,749 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:19:57,750 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:19:57,750 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@64fefc08, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:19:57,751 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:57,754 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:57,755 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:19:57,755 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca4003a connected 2024-12-05T23:19:57,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-12-05T23:19:57,887 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:19:57,887 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:19:57,888 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:19:57,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=67, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:19:57,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67 2024-12-05T23:19:57,894 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-05T23:19:57,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-05T23:19:57,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:19:57,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:19:57,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:19:57,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67 2024-12-05T23:19:58,051 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:58,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=68 2024-12-05T23:19:58,053 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-05T23:19:58,053 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-05T23:19:58,053 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-05T23:19:58,053 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:19:58,053 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca4003a, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:58,054 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca4003a, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:58,159 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca4003a, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:58,159 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=68}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x75abc9b3 to 127.0.0.1:50903 2024-12-05T23:19:58,159 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=68}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:19:58,159 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-05T23:19:58,162 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=68}] zookeeper.ZKUtil(111): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:19:58,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:19:58,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:19:58,165 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:19:58,166 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=68}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:19:58,166 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=68 2024-12-05T23:19:58,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=68 2024-12-05T23:19:58,170 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:19:58,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=68, resume processing ppid=67 2024-12-05T23:19:58,172 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=68, ppid=67, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 272 msec 2024-12-05T23:19:58,177 DEBUG [PEWorker-2 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:19:58,178 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-05T23:19:58,179 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=67, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 290 msec 2024-12-05T23:19:58,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67 2024-12-05T23:19:58,207 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:19:58,209 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:19:58,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=69, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:19:58,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-12-05T23:19:58,212 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:19:58,218 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=69, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:19:58,221 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=69, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=10 msec 2024-12-05T23:19:58,221 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:19:58,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-12-05T23:19:58,317 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-05T23:19:58,318 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:19:58,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=70, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:19:58,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=70 2024-12-05T23:19:58,320 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:19:58,321 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=70, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:19:58,323 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=70, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-05T23:19:58,323 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:19:58,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=70 2024-12-05T23:19:58,428 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:19:58,471 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[0] Thread=219 (was 219), OpenFileDescriptor=444 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=688 (was 618) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2880 (was 2903) 2024-12-05T23:19:58,498 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[0] Thread=219, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=688, ProcessCount=11, AvailableMemoryMB=2880 2024-12-05T23:19:58,500 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:19:58,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=71, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:19:58,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-12-05T23:19:58,524 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:58,524 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:58,528 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:58,529 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4003b connected 2024-12-05T23:19:58,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-12-05T23:19:58,631 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4003b, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:58,641 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:19:58,794 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:58,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=72 2024-12-05T23:19:58,797 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-05T23:19:58,825 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=72}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:58,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-12-05T23:19:58,841 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:19:58,850 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=72 2024-12-05T23:19:58,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=72 2024-12-05T23:19:58,854 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:19:58,857 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=72, resume processing ppid=71 2024-12-05T23:19:58,857 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=72, ppid=71, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 214 msec 2024-12-05T23:19:58,857 INFO [PEWorker-3 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-05T23:19:58,859 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=71, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 357 msec 2024-12-05T23:19:58,889 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x48dd01d3 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1656d4e4 2024-12-05T23:19:58,913 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:19:58,914 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:19:58,915 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@642ef926, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:19:58,916 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:58,941 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:58,941 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:19:58,944 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca4003d connected 2024-12-05T23:19:59,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-12-05T23:19:59,137 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:19:59,137 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-05T23:19:59,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=73, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:19:59,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-12-05T23:19:59,165 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:59,166 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:59,170 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:59,173 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4003e connected 2024-12-05T23:19:59,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-12-05T23:19:59,275 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4003e, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:59,286 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:19:59,430 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T23:19:59,438 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:19:59,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=74 2024-12-05T23:19:59,439 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:19:59,441 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:19:59,441 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:19:59,441 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:19:59,441 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca4003d, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:59,442 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca4003d, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:19:59,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-12-05T23:19:59,545 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca4003d, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:59,546 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=74}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x48dd01d3 to 127.0.0.1:50903 2024-12-05T23:19:59,546 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=74}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:19:59,567 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:19:59,568 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:19:59,570 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=74 2024-12-05T23:19:59,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=74 2024-12-05T23:19:59,585 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:19:59,589 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=74, resume processing ppid=73 2024-12-05T23:19:59,590 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=74, ppid=73, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 299 msec 2024-12-05T23:19:59,590 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-05T23:19:59,592 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=73, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 453 msec 2024-12-05T23:19:59,614 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1b532f1c to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@59ce2296 2024-12-05T23:19:59,632 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:19:59,633 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:19:59,633 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3f3b6cf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:19:59,635 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:59,638 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:59,639 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:19:59,640 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40040 connected 2024-12-05T23:19:59,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-12-05T23:19:59,797 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:19:59,797 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:19:59,798 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-12-05T23:19:59,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=75, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:19:59,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-12-05T23:19:59,826 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:19:59,826 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:19:59,830 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:19:59,841 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40041 connected 2024-12-05T23:19:59,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-12-05T23:19:59,953 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40041, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:19:59,957 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:00,111 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:00,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=76 2024-12-05T23:20:00,112 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:00,113 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:00,113 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:00,113 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:00,114 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40040, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:00,114 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40040, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:00,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-12-05T23:20:00,217 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40040, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:00,217 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=76}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1b532f1c to 127.0.0.1:50903 2024-12-05T23:20:00,217 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=76}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:00,235 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:00,235 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:00,244 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=76 2024-12-05T23:20:00,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=76 2024-12-05T23:20:00,248 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:00,250 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=76, resume processing ppid=75 2024-12-05T23:20:00,250 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-12-05T23:20:00,250 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=76, ppid=75, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 291 msec 2024-12-05T23:20:00,252 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=75, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 452 msec 2024-12-05T23:20:00,284 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7a12b74b to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5e82eb54 2024-12-05T23:20:00,289 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:00,290 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:00,290 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@2429c79e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:00,291 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:00,295 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:00,295 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:00,296 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40043 connected 2024-12-05T23:20:00,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-12-05T23:20:00,427 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:00,428 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:00,429 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:00,430 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-05T23:20:00,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=77, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-12-05T23:20:00,453 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:00,454 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:00,460 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:00,461 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40044 connected 2024-12-05T23:20:00,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-12-05T23:20:00,563 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40044, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:00,568 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:00,720 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:00,721 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=78 2024-12-05T23:20:00,721 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:00,722 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:00,722 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:00,723 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:00,723 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40043, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:00,723 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40043, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:00,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-12-05T23:20:00,828 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40043, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:00,829 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=78}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7a12b74b to 127.0.0.1:50903 2024-12-05T23:20:00,829 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=78}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:00,853 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:00,853 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:00,854 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=78 2024-12-05T23:20:00,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=78 2024-12-05T23:20:00,860 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:00,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=78, resume processing ppid=77 2024-12-05T23:20:00,862 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-05T23:20:00,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=78, ppid=77, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 292 msec 2024-12-05T23:20:00,864 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=77, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 432 msec 2024-12-05T23:20:00,896 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3c6c7fe5 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@311dd8a 2024-12-05T23:20:00,900 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:00,900 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:00,900 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@6d545b8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:00,901 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:00,905 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:00,905 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:00,905 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40046 connected 2024-12-05T23:20:01,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-12-05T23:20:01,077 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:01,077 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:01,078 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:01,080 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-05T23:20:01,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=79, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:01,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-12-05T23:20:01,105 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:01,106 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:01,112 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:01,112 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40047 connected 2024-12-05T23:20:01,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-12-05T23:20:01,214 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40047, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:01,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:01,376 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:01,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=80 2024-12-05T23:20:01,381 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:01,382 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:01,382 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:01,383 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:01,383 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40046, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:01,383 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40046, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:01,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-12-05T23:20:01,486 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40046, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:01,486 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=80}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3c6c7fe5 to 127.0.0.1:50903 2024-12-05T23:20:01,486 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=80}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:01,508 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:01,508 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:01,509 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=80 2024-12-05T23:20:01,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=80 2024-12-05T23:20:01,512 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:01,515 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=80, resume processing ppid=79 2024-12-05T23:20:01,515 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=80, ppid=79, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 293 msec 2024-12-05T23:20:01,515 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-05T23:20:01,518 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=79, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 435 msec 2024-12-05T23:20:01,551 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x28504ca9 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@10fe6fcf 2024-12-05T23:20:01,566 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:01,566 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:01,567 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3b0f50f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:01,568 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:01,573 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:01,573 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:01,576 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40049 connected 2024-12-05T23:20:01,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-12-05T23:20:01,707 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:01,707 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:01,708 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:01,710 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-05T23:20:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=81, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:01,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-12-05T23:20:01,729 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:01,730 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:01,733 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:01,733 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4004a connected 2024-12-05T23:20:01,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-12-05T23:20:01,837 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4004a, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:01,842 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:01,994 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:01,995 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=82 2024-12-05T23:20:01,995 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:01,996 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:01,996 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:01,996 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:01,997 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40049, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:01,997 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40049, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:02,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-12-05T23:20:02,099 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40049, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:02,099 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=82}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x28504ca9 to 127.0.0.1:50903 2024-12-05T23:20:02,099 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=82}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:02,120 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:02,121 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:02,121 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=82 2024-12-05T23:20:02,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=82 2024-12-05T23:20:02,124 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:02,127 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=82, resume processing ppid=81 2024-12-05T23:20:02,127 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-05T23:20:02,127 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=82, ppid=81, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 283 msec 2024-12-05T23:20:02,129 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=81, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 417 msec 2024-12-05T23:20:02,156 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5300cba4 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@765e9241 2024-12-05T23:20:02,164 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:02,164 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:02,165 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@7db8ba6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:02,166 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:02,168 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:02,169 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:02,169 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca4004c connected 2024-12-05T23:20:02,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-12-05T23:20:02,337 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:02,337 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:02,338 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:02,340 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-05T23:20:02,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=83, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:02,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-12-05T23:20:02,362 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:02,363 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:02,366 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:02,367 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4004d connected 2024-12-05T23:20:02,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-12-05T23:20:02,468 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4004d, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:02,475 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:02,627 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:02,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=84 2024-12-05T23:20:02,632 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:02,634 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:02,634 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:02,634 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:02,634 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca4004c, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:02,635 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca4004c, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:02,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-12-05T23:20:02,752 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca4004c, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:02,752 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=84}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5300cba4 to 127.0.0.1:50903 2024-12-05T23:20:02,752 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=84}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:02,774 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:02,775 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:02,775 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=84 2024-12-05T23:20:02,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=84 2024-12-05T23:20:02,778 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:02,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=84, resume processing ppid=83 2024-12-05T23:20:02,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=84, ppid=83, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 303 msec 2024-12-05T23:20:02,781 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-05T23:20:02,783 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=83, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 441 msec 2024-12-05T23:20:02,823 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2400d6f5 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@d1fda90 2024-12-05T23:20:02,827 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:02,828 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:02,828 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@29340ff0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:02,829 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:02,833 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:02,834 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:02,834 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca4004f connected 2024-12-05T23:20:02,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-12-05T23:20:02,967 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:02,970 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:02,972 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-05T23:20:02,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=85, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-12-05T23:20:03,025 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:03,026 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:03,033 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:03,034 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40050 connected 2024-12-05T23:20:03,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-12-05T23:20:03,136 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40050, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:03,141 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:03,294 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:03,295 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=86 2024-12-05T23:20:03,295 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-12-05T23:20:03,297 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=86 2024-12-05T23:20:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=86 2024-12-05T23:20:03,301 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:03,304 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=86, resume processing ppid=85 2024-12-05T23:20:03,304 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=86, ppid=85, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 160 msec 2024-12-05T23:20:03,304 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-05T23:20:03,307 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=85, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 332 msec 2024-12-05T23:20:03,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-12-05T23:20:03,607 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:03,608 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:03,609 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:03,610 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-12-05T23:20:03,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=87, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:03,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-12-05T23:20:03,639 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:03,639 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:03,643 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:03,643 ERROR [PEWorker-2-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$360/0x00007f9a60914e50@4814b4bc rejected from java.util.concurrent.ThreadPoolExecutor@4c2b3966[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 0] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.PendingWatcher.process(PendingWatcher.java:38) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-12-05T23:20:03,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-12-05T23:20:03,745 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:03,750 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:03,835 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 2000 millis 2024-12-05T23:20:03,902 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:03,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=88 2024-12-05T23:20:03,903 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:03,904 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:03,905 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:03,905 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:03,905 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca4004f, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:03,905 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca4004f, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:03,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-12-05T23:20:04,011 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca4004f, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:04,012 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=88}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2400d6f5 to 127.0.0.1:50903 2024-12-05T23:20:04,012 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=88}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:04,035 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:04,036 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:04,040 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-12-05T23:20:04,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=88 2024-12-05T23:20:04,043 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:04,045 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=88, resume processing ppid=87 2024-12-05T23:20:04,045 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=88, ppid=87, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 294 msec 2024-12-05T23:20:04,045 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-12-05T23:20:04,047 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=87, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 436 msec 2024-12-05T23:20:04,099 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7945a5f3 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@494826a3 2024-12-05T23:20:04,120 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:04,121 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:04,121 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@22689c16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:04,133 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:04,163 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:04,164 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:04,165 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40053 connected 2024-12-05T23:20:04,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-12-05T23:20:04,237 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:04,238 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:04,240 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-12-05T23:20:04,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=89, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:04,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-12-05T23:20:04,262 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:04,263 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:04,268 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:04,268 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40054 connected 2024-12-05T23:20:04,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-12-05T23:20:04,371 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40054, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:04,384 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:04,530 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T23:20:04,536 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:04,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=90 2024-12-05T23:20:04,537 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:04,539 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:04,539 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:04,542 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:04,543 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40053, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:04,543 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40053, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:04,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-12-05T23:20:04,646 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40053, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:04,646 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=90}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7945a5f3 to 127.0.0.1:50903 2024-12-05T23:20:04,646 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=90}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:04,671 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:04,671 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:04,671 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-12-05T23:20:04,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=90 2024-12-05T23:20:04,675 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:04,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=90, resume processing ppid=89 2024-12-05T23:20:04,677 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-12-05T23:20:04,678 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=90, ppid=89, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 291 msec 2024-12-05T23:20:04,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=89, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 438 msec 2024-12-05T23:20:04,713 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3c68609b to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@79f2b521 2024-12-05T23:20:04,741 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:04,742 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:04,742 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@58ea825e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:04,743 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:04,751 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:04,751 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:04,752 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40056 connected 2024-12-05T23:20:04,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-12-05T23:20:04,867 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:04,867 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:04,868 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:04,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=91, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:04,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91 2024-12-05T23:20:04,872 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:04,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:04,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:04,875 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:04,876 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:04,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91 2024-12-05T23:20:05,028 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:05,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=92 2024-12-05T23:20:05,029 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-05T23:20:05,029 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-05T23:20:05,029 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-05T23:20:05,029 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:05,029 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40056, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:05,029 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40056, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:05,132 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40056, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:05,132 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=92}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3c68609b to 127.0.0.1:50903 2024-12-05T23:20:05,132 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=92}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:05,132 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-05T23:20:05,135 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=92}] zookeeper.ZKUtil(111): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:05,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:05,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:05,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:05,140 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=92}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:05,140 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-12-05T23:20:05,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=92 2024-12-05T23:20:05,145 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:05,147 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=92, resume processing ppid=91 2024-12-05T23:20:05,147 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=92, ppid=91, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 269 msec 2024-12-05T23:20:05,152 DEBUG [PEWorker-4 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:05,152 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-05T23:20:05,154 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=91, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 284 msec 2024-12-05T23:20:05,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91 2024-12-05T23:20:05,187 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:05,188 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:05,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=93, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:05,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=93 2024-12-05T23:20:05,190 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:05,191 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=93, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:05,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=93, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-05T23:20:05,192 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:05,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=93 2024-12-05T23:20:05,297 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-05T23:20:05,298 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:20:05,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=94, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:05,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=94 2024-12-05T23:20:05,300 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:05,301 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=94, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:05,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=94, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-05T23:20:05,302 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:05,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=94 2024-12-05T23:20:05,408 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:20:05,423 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[0] Thread=220 (was 219) - Thread LEAK? -, OpenFileDescriptor=446 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=665 (was 688), ProcessCount=11 (was 11), AvailableMemoryMB=2736 (was 2880) 2024-12-05T23:20:05,436 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[0] Thread=220, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=665, ProcessCount=11, AvailableMemoryMB=2736 2024-12-05T23:20:05,437 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:20:05,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=95, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:20:05,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=95 2024-12-05T23:20:05,467 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:05,468 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:05,472 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:05,472 ERROR [PEWorker-5-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$360/0x00007f9a60914e50@10529e3a rejected from java.util.concurrent.ThreadPoolExecutor@146d18e8[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 0] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.PendingWatcher.process(PendingWatcher.java:38) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-12-05T23:20:05,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=95 2024-12-05T23:20:05,574 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:05,582 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:05,734 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:05,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=96 2024-12-05T23:20:05,735 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-05T23:20:05,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=95 2024-12-05T23:20:05,765 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=96}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:05,781 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:05,795 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-05T23:20:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=96 2024-12-05T23:20:05,797 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:05,799 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=96, resume processing ppid=95 2024-12-05T23:20:05,799 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=96, ppid=95, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 215 msec 2024-12-05T23:20:05,799 INFO [PEWorker-1 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-05T23:20:05,801 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=95, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 362 msec 2024-12-05T23:20:05,836 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5db75cc3 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@209d22b5 2024-12-05T23:20:05,840 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:05,841 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:05,841 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@64d095e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:05,842 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:05,846 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:05,847 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:05,847 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40059 connected 2024-12-05T23:20:05,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-12-05T23:20:06,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=95 2024-12-05T23:20:06,066 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:06,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-05T23:20:06,072 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3938): Client=jenkins//172.17.0.2 disable replication peer, id=1 2024-12-05T23:20:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=97, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure 2024-12-05T23:20:06,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=97 2024-12-05T23:20:06,087 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:06,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=97 2024-12-05T23:20:06,239 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:06,240 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=98 2024-12-05T23:20:06,240 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=DISABLE_PEER 2024-12-05T23:20:06,241 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:06,241 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:06,241 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:06,242 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40059, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:06,242 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40059, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:06,344 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40059, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:06,345 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=98}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5db75cc3 to 127.0.0.1:50903 2024-12-05T23:20:06,345 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=98}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:06,364 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:06,364 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:06,365 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-05T23:20:06,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=98 2024-12-05T23:20:06,369 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for DISABLE on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:06,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=98, resume processing ppid=97 2024-12-05T23:20:06,371 INFO [PEWorker-4 {}] replication.DisablePeerProcedure(67): Successfully disabled peer 1 2024-12-05T23:20:06,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=98, ppid=97, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 282 msec 2024-12-05T23:20:06,372 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=97, state=SUCCESS; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure in 299 msec 2024-12-05T23:20:06,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=97 2024-12-05T23:20:06,398 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: DISABLE_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:06,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-05T23:20:06,399 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:06,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=99, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:06,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=99 2024-12-05T23:20:06,404 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:06,408 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0c42c003 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4cdeeb4f 2024-12-05T23:20:06,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:06,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:06,411 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:06,412 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:06,427 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:06,428 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:06,428 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@52f3d465, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:06,429 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:06,444 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:06,444 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:06,444 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca4005b connected 2024-12-05T23:20:06,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=99 2024-12-05T23:20:06,564 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:06,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=100 2024-12-05T23:20:06,565 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-05T23:20:06,565 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-05T23:20:06,565 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-05T23:20:06,565 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:06,565 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca4005b, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:06,565 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca4005b, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:06,670 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca4005b, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:06,670 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=100}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0c42c003 to 127.0.0.1:50903 2024-12-05T23:20:06,670 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=100}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:06,670 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-05T23:20:06,673 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=100}] zookeeper.ZKUtil(111): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:06,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:06,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:06,677 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:06,677 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=100}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:06,678 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-05T23:20:06,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=100 2024-12-05T23:20:06,683 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:06,685 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=100, resume processing ppid=99 2024-12-05T23:20:06,685 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=100, ppid=99, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 271 msec 2024-12-05T23:20:06,689 DEBUG [PEWorker-5 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:06,689 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-05T23:20:06,691 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=99, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 290 msec 2024-12-05T23:20:06,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=99 2024-12-05T23:20:06,717 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:06,717 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:06,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=101, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:06,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=101 2024-12-05T23:20:06,719 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:06,722 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=101, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:06,723 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=101, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-05T23:20:06,723 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:06,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=101 2024-12-05T23:20:06,827 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-05T23:20:06,828 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:20:06,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=102, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:06,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=102 2024-12-05T23:20:06,830 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:06,831 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=102, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:06,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=102, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-05T23:20:06,832 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:06,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=102 2024-12-05T23:20:06,938 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:20:06,961 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[0] Thread=220 (was 220), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=652 (was 665), ProcessCount=11 (was 11), AvailableMemoryMB=2703 (was 2736) 2024-12-05T23:20:07,000 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[0] Thread=220, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=652, ProcessCount=11, AvailableMemoryMB=2698 2024-12-05T23:20:07,001 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=dummy_peer 2024-12-05T23:20:07,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=103, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:07,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=103 2024-12-05T23:20:07,004 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer dummy_peer, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:07,005 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=103, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:07,007 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=103, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-05T23:20:07,007 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:07,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=103 2024-12-05T23:20:07,107 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: dummy_peer failed with dummy_peer 2024-12-05T23:20:07,108 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:07,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=104, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:07,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=104 2024-12-05T23:20:07,110 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:07,112 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=104, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:07,113 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=104, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-05T23:20:07,113 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:07,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=104 2024-12-05T23:20:07,217 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-05T23:20:07,218 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:20:07,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=105, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:07,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=105 2024-12-05T23:20:07,220 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:07,221 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=105, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:07,222 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=105, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-05T23:20:07,222 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=105 2024-12-05T23:20:07,332 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:20:07,348 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[0] Thread=220 (was 220), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=652 (was 652), ProcessCount=11 (was 11), AvailableMemoryMB=2696 (was 2698) 2024-12-05T23:20:07,359 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerConfig[0] Thread=220, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=652, ProcessCount=11, AvailableMemoryMB=2695 2024-12-05T23:20:07,360 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:20:07,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=106, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:20:07,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-12-05T23:20:07,384 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:07,385 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:07,389 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:07,390 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4005c connected 2024-12-05T23:20:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-12-05T23:20:07,492 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4005c, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:07,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:07,653 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:07,654 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=107 2024-12-05T23:20:07,654 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-05T23:20:07,672 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=107}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:07,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-12-05T23:20:07,695 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:07,709 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=107 2024-12-05T23:20:07,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=107 2024-12-05T23:20:07,713 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:07,715 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=107, resume processing ppid=106 2024-12-05T23:20:07,715 INFO [PEWorker-3 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-05T23:20:07,715 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=107, ppid=106, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 212 msec 2024-12-05T23:20:07,716 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=106, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 355 msec 2024-12-05T23:20:07,759 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0f880cbc to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@79f3edae 2024-12-05T23:20:07,763 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:07,764 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:07,764 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@566f2eb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:07,767 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:07,779 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:07,780 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca4005e connected 2024-12-05T23:20:07,780 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:07,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-12-05T23:20:07,986 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:07,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-05T23:20:07,987 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:07,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=108, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:07,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-12-05T23:20:07,992 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:07,996 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:07,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:07,997 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:07,997 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:08,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-12-05T23:20:08,149 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:08,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=109 2024-12-05T23:20:08,150 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-05T23:20:08,150 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-05T23:20:08,150 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-05T23:20:08,150 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:08,150 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca4005e, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:08,151 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca4005e, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:08,253 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca4005e, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:08,253 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=109}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0f880cbc to 127.0.0.1:50903 2024-12-05T23:20:08,253 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=109}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:08,253 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-05T23:20:08,255 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=109}] zookeeper.ZKUtil(111): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:08,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:08,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:08,261 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:08,261 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=109}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:08,261 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=109 2024-12-05T23:20:08,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=109 2024-12-05T23:20:08,264 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:08,266 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=109, resume processing ppid=108 2024-12-05T23:20:08,266 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=109, ppid=108, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 267 msec 2024-12-05T23:20:08,271 DEBUG [PEWorker-2 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:08,271 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-05T23:20:08,273 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=108, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 283 msec 2024-12-05T23:20:08,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-12-05T23:20:08,307 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:08,308 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:08,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=110, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:08,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-12-05T23:20:08,310 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:08,312 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=110, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:08,313 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=110, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-05T23:20:08,313 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:08,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-12-05T23:20:08,417 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-05T23:20:08,418 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:20:08,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=111, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:08,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=111 2024-12-05T23:20:08,420 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:08,421 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=111, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:08,422 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=111, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-05T23:20:08,422 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:08,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=111 2024-12-05T23:20:08,527 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:20:08,542 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerConfig[0] Thread=220 (was 220), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=652 (was 652), ProcessCount=11 (was 11), AvailableMemoryMB=2660 (was 2695) 2024-12-05T23:20:08,553 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[0] Thread=220, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=652, ProcessCount=11, AvailableMemoryMB=2659 2024-12-05T23:20:08,559 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:20:08,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-05T23:20:08,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] ipc.CallRunner(138): callId: 348 service: MasterService methodName: AddReplicationPeer size: 80 connection: 172.17.0.2:33460 deadline: 1733440868559, exception=java.io.IOException: Replication peer modification disabled 2024-12-05T23:20:08,581 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(168): Call to master failed, tries = 1, maxAttempts = 2, timeout = 120000 ms, time elapsed = 21 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:245) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:431) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 33 more 2024-12-05T23:20:08,686 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:20:08,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-05T23:20:08,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] ipc.CallRunner(138): callId: 349 service: MasterService methodName: AddReplicationPeer size: 80 connection: 172.17.0.2:33460 deadline: 1733440868686, exception=java.io.IOException: Replication peer modification disabled 2024-12-05T23:20:08,687 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(168): Call to master failed, tries = 2, maxAttempts = 2, timeout = 120000 ms, time elapsed = 127 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:245) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:431) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 33 more 2024-12-05T23:20:08,688 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Failed after attempts=2, exceptions: 2024-12-05T23:20:08.582Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) 2024-12-05T23:20:08.687Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) 2024-12-05T23:20:08,690 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:08,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=112, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:08,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-12-05T23:20:08,691 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:08,692 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=112, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:08,692 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=112, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-05T23:20:08,692 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:08,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-12-05T23:20:08,796 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-05T23:20:08,797 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:20:08,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=113, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:08,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=113 2024-12-05T23:20:08,799 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:08,800 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=113, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:08,801 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=113, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-05T23:20:08,801 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:08,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=113 2024-12-05T23:20:08,907 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:20:08,919 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[0] Thread=220 (was 220), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=652 (was 652), ProcessCount=11 (was 11), AvailableMemoryMB=2655 (was 2659) 2024-12-05T23:20:08,929 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[0] Thread=220, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=652, ProcessCount=11, AvailableMemoryMB=2655 2024-12-05T23:20:08,930 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:20:08,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=114, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:20:08,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-12-05T23:20:08,947 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:08,948 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:08,950 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:08,951 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4005f connected 2024-12-05T23:20:09,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-12-05T23:20:09,054 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4005f, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:09,062 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:09,214 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:09,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=115 2024-12-05T23:20:09,215 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-05T23:20:09,231 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=115}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-12-05T23:20:09,247 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:09,257 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=115 2024-12-05T23:20:09,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=115 2024-12-05T23:20:09,259 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:09,261 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=115, resume processing ppid=114 2024-12-05T23:20:09,261 INFO [PEWorker-4 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-05T23:20:09,261 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=115, ppid=114, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 197 msec 2024-12-05T23:20:09,262 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=114, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 331 msec 2024-12-05T23:20:09,288 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0aa977b5 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@10dfad9e 2024-12-05T23:20:09,291 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:09,292 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:09,292 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@6db5738c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:09,293 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:09,295 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:09,295 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:09,295 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40061 connected 2024-12-05T23:20:09,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-12-05T23:20:09,557 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:09,557 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:09,558 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:09,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=116, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:09,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116 2024-12-05T23:20:09,561 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:09,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:09,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:09,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:09,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:09,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116 2024-12-05T23:20:09,717 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:09,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=117 2024-12-05T23:20:09,718 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-05T23:20:09,718 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-05T23:20:09,718 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-05T23:20:09,718 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:09,718 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40061, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:09,719 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40061, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:09,822 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40061, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:09,822 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=117}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0aa977b5 to 127.0.0.1:50903 2024-12-05T23:20:09,822 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=117}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:09,822 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-05T23:20:09,824 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=117}] zookeeper.ZKUtil(111): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:09,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:09,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:09,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:09,827 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=117}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:09,827 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=117 2024-12-05T23:20:09,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=117 2024-12-05T23:20:09,829 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:09,830 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=117, resume processing ppid=116 2024-12-05T23:20:09,830 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=117, ppid=116, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 264 msec 2024-12-05T23:20:09,835 DEBUG [PEWorker-5 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:09,835 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-05T23:20:09,836 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=116, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 277 msec 2024-12-05T23:20:09,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116 2024-12-05T23:20:09,876 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:09,877 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:20:09,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=118, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:09,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=118 2024-12-05T23:20:09,879 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:09,879 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=118, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:09,880 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=118, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-05T23:20:09,880 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:09,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=118 2024-12-05T23:20:09,987 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:20:10,000 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[0] Thread=220 (was 220), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=652 (was 652), ProcessCount=11 (was 11), AvailableMemoryMB=2642 (was 2655) 2024-12-05T23:20:10,011 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerBandwidth[0] Thread=220, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=652, ProcessCount=11, AvailableMemoryMB=2641 2024-12-05T23:20:10,012 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:20:10,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=119, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:20:10,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119 2024-12-05T23:20:10,030 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:10,031 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:10,034 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:10,034 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40062 connected 2024-12-05T23:20:10,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119 2024-12-05T23:20:10,136 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40062, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:10,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:10,297 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:10,298 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=120 2024-12-05T23:20:10,298 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-05T23:20:10,314 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=120}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:10,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119 2024-12-05T23:20:10,329 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:10,341 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-05T23:20:10,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=120 2024-12-05T23:20:10,343 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:10,344 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=120, resume processing ppid=119 2024-12-05T23:20:10,344 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=120, ppid=119, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 198 msec 2024-12-05T23:20:10,344 INFO [PEWorker-5 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-05T23:20:10,345 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=119, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 332 msec 2024-12-05T23:20:10,372 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x62011c74 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2c61166d 2024-12-05T23:20:10,375 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:10,375 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:10,376 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@9719ad0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:10,376 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:10,380 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:10,380 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:10,380 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40064 connected 2024-12-05T23:20:10,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119 2024-12-05T23:20:10,636 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:10,637 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:10,638 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-12-05T23:20:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=121, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=121 2024-12-05T23:20:10,650 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T23:20:10,655 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:10,655 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:10,658 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:10,658 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40065 connected 2024-12-05T23:20:10,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=121 2024-12-05T23:20:10,760 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40065, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:10,764 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:10,916 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:10,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=122 2024-12-05T23:20:10,916 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:10,917 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-05T23:20:10,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=122 2024-12-05T23:20:10,919 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:10,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=122, resume processing ppid=121 2024-12-05T23:20:10,921 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-12-05T23:20:10,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=122, ppid=121, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 156 msec 2024-12-05T23:20:10,922 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=121, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 283 msec 2024-12-05T23:20:10,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=121 2024-12-05T23:20:10,957 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:10,957 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:10,958 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:10,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=123, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:10,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=123 2024-12-05T23:20:10,960 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:10,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:10,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:10,970 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:10,971 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:11,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=123 2024-12-05T23:20:11,122 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:11,123 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=124 2024-12-05T23:20:11,123 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-05T23:20:11,123 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-05T23:20:11,123 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-05T23:20:11,123 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:11,123 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40064, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:11,124 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40064, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:11,226 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40064, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:11,226 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=124}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x62011c74 to 127.0.0.1:50903 2024-12-05T23:20:11,226 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=124}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:11,227 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-05T23:20:11,228 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=124}] zookeeper.ZKUtil(111): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:11,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:11,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:11,230 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:11,230 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=124}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:11,230 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-05T23:20:11,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=124 2024-12-05T23:20:11,232 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:11,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=124, resume processing ppid=123 2024-12-05T23:20:11,234 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=124, ppid=123, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 261 msec 2024-12-05T23:20:11,238 DEBUG [PEWorker-2 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:11,238 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-05T23:20:11,239 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=123, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 280 msec 2024-12-05T23:20:11,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=123 2024-12-05T23:20:11,276 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:11,276 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=125, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=125 2024-12-05T23:20:11,278 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:11,278 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=125, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:11,279 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=125, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-05T23:20:11,279 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:11,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=125 2024-12-05T23:20:11,387 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-05T23:20:11,388 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:20:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=126, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:11,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=126 2024-12-05T23:20:11,389 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:11,390 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=126, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:11,391 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=126, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-05T23:20:11,391 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:11,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=126 2024-12-05T23:20:11,497 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:20:11,509 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerBandwidth[0] Thread=222 (was 220) - Thread LEAK? -, OpenFileDescriptor=448 (was 446) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=600 (was 652), ProcessCount=11 (was 11), AvailableMemoryMB=2616 (was 2641) 2024-12-05T23:20:11,518 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[0] Thread=222, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=600, ProcessCount=11, AvailableMemoryMB=2615 2024-12-05T23:20:11,519 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:20:11,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=127, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:20:11,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=127 2024-12-05T23:20:11,526 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:11,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=127 2024-12-05T23:20:11,678 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:11,678 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=128 2024-12-05T23:20:11,679 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=128}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-05T23:20:11,708 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=128}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:11,717 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-05T23:20:11,718 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(587): peerId=1, queueId=1 (queues=1) is replicating from cluster=a6e07b25-fe39-42c3-aefd-b0129d0da2f3 to cluster=a6e07b25-fe39-42c3-aefd-b0129d0da2f3 2024-12-05T23:20:11,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=128 2024-12-05T23:20:11,719 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(343): peerId=1, starting shipping worker for walGroupId=9e5740704f6b%2C36187%2C1733440777223 2024-12-05T23:20:11,719 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:11,721 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=128, resume processing ppid=127 2024-12-05T23:20:11,721 INFO [PEWorker-3 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-05T23:20:11,721 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=128, ppid=127, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 193 msec 2024-12-05T23:20:11,721 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSourceWALReader(112): peerClusterZnode=1, ReplicationSourceWALReaderThread : 1 inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-12-05T23:20:11,722 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=127, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 202 msec 2024-12-05T23:20:11,723 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1.replicationSource.shipper9e5740704f6b%2C36187%2C1733440777223,1 {}] regionserver.ReplicationSourceShipper(98): Running ReplicationSourceShipper Thread for wal group: 9e5740704f6b%2C36187%2C1733440777223 2024-12-05T23:20:11,723 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1.replicationSource.wal-reader.9e5740704f6b%2C36187%2C1733440777223,1 {}] regionserver.WALEntryStream(260): Creating new reader hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/WALs/9e5740704f6b,36187,1733440777223/9e5740704f6b%2C36187%2C1733440777223.1733440780716, startPosition=0, beingWritten=true 2024-12-05T23:20:11,789 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1.replicationSource.wal-reader.9e5740704f6b%2C36187%2C1733440777223,1 {}] regionserver.ReplicationSourceWALReader(181): Read 0 WAL entries eligible for replication 2024-12-05T23:20:11,789 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1.replicationSource.wal-reader.9e5740704f6b%2C36187%2C1733440777223,1 {}] regionserver.WALEntryStream(223): Reset reader hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/WALs/9e5740704f6b,36187,1733440777223/9e5740704f6b%2C36187%2C1733440777223.1733440780716 to pos 589, reset compression=false 2024-12-05T23:20:11,789 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1.replicationSource.shipper9e5740704f6b%2C36187%2C1733440777223,1 {}] regionserver.ReplicationSourceShipper(110): Shipper from source 1 got entry batch from reader: WALEntryBatch [walEntries=[], lastWalPath=hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/WALs/9e5740704f6b,36187,1733440777223/9e5740704f6b%2C36187%2C1733440777223.1733440780716, lastWalPosition=589, nbRowKeys=0, nbHFiles=0, heapSize=0, lastSeqIds={}, endOfFile=false,usedBufferSize=0] 2024-12-05T23:20:11,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=127 2024-12-05T23:20:11,836 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:11,837 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=2, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:20:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=129, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:20:11,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=129 2024-12-05T23:20:11,839 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] ... 13 more 2024-12-05T23:20:11,839 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=129, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:11,840 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=129, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=3 msec 2024-12-05T23:20:11,840 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:11,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=129 2024-12-05T23:20:11,947 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 2 failed with Invalid cluster key: 2024-12-05T23:20:11,947 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:11,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=130, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:11,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-12-05T23:20:11,950 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:11,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:11,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:11,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:11,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:12,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-12-05T23:20:12,104 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:12,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=131 2024-12-05T23:20:12,105 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-05T23:20:12,105 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-05T23:20:12,105 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-05T23:20:12,126 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-05T23:20:12,128 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:36236, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-05T23:20:12,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-12-05T23:20:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-12-05T23:20:13,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-12-05T23:20:13,106 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1.replicationSource.wal-reader.9e5740704f6b%2C36187%2C1733440777223,1 {}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:131) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:166) ~[classes/:?] 2024-12-05T23:20:13,106 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1.replicationSource.shipper9e5740704f6b%2C36187%2C1733440777223,1 {}] regionserver.ReplicationSourceShipper(123): Interrupted while waiting for next replication entry batch java.lang.InterruptedException: null at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1681) ~[?:?] at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) ~[?:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.poll(ReplicationSourceWALReader.java:313) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceShipper.run(ReplicationSourceShipper.java:109) ~[classes/:?] 2024-12-05T23:20:14,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-12-05T23:20:14,106 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.ReplicationSource(711): peerId=1, ReplicationSourceWorker RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1.replicationSource.shipper9e5740704f6b%2C36187%2C1733440777223,1 terminated 2024-12-05T23:20:14,106 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-05T23:20:14,107 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=131}] zookeeper.ZKUtil(111): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:14,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:14,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:14,110 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:14,110 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=131}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:14,110 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=131 2024-12-05T23:20:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=131 2024-12-05T23:20:14,112 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:14,115 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=131, resume processing ppid=130 2024-12-05T23:20:14,115 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=131, ppid=130, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 2.1590 sec 2024-12-05T23:20:14,120 DEBUG [PEWorker-3 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:14,120 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-05T23:20:14,122 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=130, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 2.1730 sec 2024-12-05T23:20:15,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-12-05T23:20:16,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-12-05T23:20:16,107 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:16,107 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:20:16,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=132, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:16,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-12-05T23:20:16,109 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:16,111 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=132, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:16,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=132, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-05T23:20:16,112 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:16,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-12-05T23:20:16,218 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:20:16,236 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[0] Thread=220 (was 222), OpenFileDescriptor=451 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=576 (was 600), ProcessCount=11 (was 11), AvailableMemoryMB=3753 (was 2615) - AvailableMemoryMB LEAK? - 2024-12-05T23:20:16,249 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[1] Thread=219, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=576, ProcessCount=11, AvailableMemoryMB=3750 2024-12-05T23:20:16,251 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:20:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=133, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:20:16,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=133 2024-12-05T23:20:16,270 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:16,270 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:16,273 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:16,274 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40066 connected 2024-12-05T23:20:16,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=133 2024-12-05T23:20:16,376 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40066, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:16,383 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:16,536 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:16,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=134 2024-12-05T23:20:16,536 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-05T23:20:16,554 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=134}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=133 2024-12-05T23:20:16,570 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:16,580 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-05T23:20:16,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=134 2024-12-05T23:20:16,583 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:16,584 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=134, resume processing ppid=133 2024-12-05T23:20:16,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=134, ppid=133, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 200 msec 2024-12-05T23:20:16,585 INFO [PEWorker-2 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-05T23:20:16,586 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=133, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 334 msec 2024-12-05T23:20:16,612 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x057a6435 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1b3b41b7 2024-12-05T23:20:16,615 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:16,616 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:16,616 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@517b49ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:16,617 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:16,619 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:16,620 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:16,620 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40068 connected 2024-12-05T23:20:16,729 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T23:20:16,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=133 2024-12-05T23:20:16,877 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:16,878 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-05T23:20:16,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=135, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:16,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=135 2024-12-05T23:20:16,895 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:16,895 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:16,904 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:16,904 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40069 connected 2024-12-05T23:20:16,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=135 2024-12-05T23:20:17,006 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40069, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:17,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:17,162 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:17,162 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=136 2024-12-05T23:20:17,163 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:17,164 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:17,164 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:17,164 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:17,164 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40068, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:17,164 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40068, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:17,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=135 2024-12-05T23:20:17,267 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40068, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:17,267 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=136}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x057a6435 to 127.0.0.1:50903 2024-12-05T23:20:17,267 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=136}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:17,282 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:17,283 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:17,283 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-05T23:20:17,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=136 2024-12-05T23:20:17,285 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:17,286 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=136, resume processing ppid=135 2024-12-05T23:20:17,286 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-05T23:20:17,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=136, ppid=135, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 275 msec 2024-12-05T23:20:17,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=135, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 409 msec 2024-12-05T23:20:17,314 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3818e6eb to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@52574745 2024-12-05T23:20:17,318 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:17,318 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:17,318 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3c93df9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:17,320 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:17,322 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:17,322 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:17,322 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca4006b connected 2024-12-05T23:20:17,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=135 2024-12-05T23:20:17,507 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:17,507 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:17,508 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:17,509 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:17,509 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-05T23:20:17,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=137, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:17,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=137 2024-12-05T23:20:17,527 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:17,527 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:17,530 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:17,530 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4006c connected 2024-12-05T23:20:17,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=137 2024-12-05T23:20:17,634 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4006c, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:17,638 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:17,790 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:17,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=138 2024-12-05T23:20:17,790 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:17,792 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:17,792 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:17,792 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:17,792 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca4006b, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:17,792 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca4006b, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:17,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=137 2024-12-05T23:20:17,894 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca4006b, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:17,894 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=138}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3818e6eb to 127.0.0.1:50903 2024-12-05T23:20:17,895 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=138}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:17,917 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:17,918 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:17,918 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-12-05T23:20:17,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=138 2024-12-05T23:20:17,921 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:17,922 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=138, resume processing ppid=137 2024-12-05T23:20:17,923 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-05T23:20:17,923 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=138, ppid=137, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 283 msec 2024-12-05T23:20:17,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=137, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 413 msec 2024-12-05T23:20:17,952 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x283e93f0 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@32bb6155 2024-12-05T23:20:17,957 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:17,957 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:17,957 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@69069de4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:17,958 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:17,961 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:17,961 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:17,962 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca4006e connected 2024-12-05T23:20:18,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=137 2024-12-05T23:20:18,136 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:18,137 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:18,138 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:18,138 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:18,139 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:18,140 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-05T23:20:18,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=139, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:18,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139 2024-12-05T23:20:18,157 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:18,157 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:18,160 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:18,160 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4006f connected 2024-12-05T23:20:18,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139 2024-12-05T23:20:18,262 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4006f, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:18,267 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:18,419 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:18,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=140 2024-12-05T23:20:18,420 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:18,421 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:18,421 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:18,421 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:18,422 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca4006e, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:18,422 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca4006e, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:18,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139 2024-12-05T23:20:18,524 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca4006e, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:18,524 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=140}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x283e93f0 to 127.0.0.1:50903 2024-12-05T23:20:18,524 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=140}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:18,547 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:18,548 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:18,548 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-12-05T23:20:18,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=140 2024-12-05T23:20:18,550 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:18,552 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=140, resume processing ppid=139 2024-12-05T23:20:18,552 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-05T23:20:18,552 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=140, ppid=139, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 284 msec 2024-12-05T23:20:18,554 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=139, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 412 msec 2024-12-05T23:20:18,582 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x58861b18 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@14b9188b 2024-12-05T23:20:18,586 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:18,587 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:18,587 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@d82c9dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:18,588 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:18,592 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:18,592 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:18,592 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40071 connected 2024-12-05T23:20:18,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139 2024-12-05T23:20:18,767 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:18,767 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:18,768 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:18,768 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:18,769 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-05T23:20:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=141, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:18,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=141 2024-12-05T23:20:18,785 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:18,786 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:18,788 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:18,788 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40072 connected 2024-12-05T23:20:18,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=141 2024-12-05T23:20:18,890 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40072, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:18,893 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:19,044 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:19,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=142 2024-12-05T23:20:19,045 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:19,046 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:19,046 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:19,046 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:19,046 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40071, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:19,046 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40071, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=141 2024-12-05T23:20:19,156 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40071, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:19,156 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=142}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x58861b18 to 127.0.0.1:50903 2024-12-05T23:20:19,156 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=142}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:19,202 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:19,203 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:19,205 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-12-05T23:20:19,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=142 2024-12-05T23:20:19,207 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:19,209 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=142, resume processing ppid=141 2024-12-05T23:20:19,209 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=142, ppid=141, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 314 msec 2024-12-05T23:20:19,209 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-05T23:20:19,210 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=141, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 440 msec 2024-12-05T23:20:19,246 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x792f0700 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@732dfc9c 2024-12-05T23:20:19,249 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:19,250 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:19,250 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@1096d901, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:19,250 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:19,253 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:19,253 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:19,253 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40074 connected 2024-12-05T23:20:19,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=141 2024-12-05T23:20:19,397 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:19,397 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:19,398 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:19,398 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-05T23:20:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=143, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:19,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143 2024-12-05T23:20:19,414 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:19,414 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:19,419 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:19,419 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40075 connected 2024-12-05T23:20:19,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143 2024-12-05T23:20:19,521 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40075, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:19,524 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:19,676 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:19,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=144 2024-12-05T23:20:19,677 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:19,678 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:19,678 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:19,678 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:19,679 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40074, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:19,679 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40074, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:19,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143 2024-12-05T23:20:19,781 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40074, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:19,781 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=144}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x792f0700 to 127.0.0.1:50903 2024-12-05T23:20:19,781 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=144}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:19,796 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:19,796 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:19,797 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-12-05T23:20:19,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=144 2024-12-05T23:20:19,799 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:19,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=144, resume processing ppid=143 2024-12-05T23:20:19,800 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-05T23:20:19,800 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=144, ppid=143, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 275 msec 2024-12-05T23:20:19,801 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=143, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 402 msec 2024-12-05T23:20:19,823 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x33d41122 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@56670b60 2024-12-05T23:20:19,826 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:19,827 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:19,827 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@53bb8828, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:19,827 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:19,830 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:19,830 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:19,830 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40077 connected 2024-12-05T23:20:20,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143 2024-12-05T23:20:20,026 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:20,026 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:20,027 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-05T23:20:20,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=145, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:20,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=145 2024-12-05T23:20:20,043 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:20,043 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:20,053 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:20,054 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40078 connected 2024-12-05T23:20:20,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=145 2024-12-05T23:20:20,164 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40078, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:20,167 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:20,319 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:20,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=146 2024-12-05T23:20:20,320 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:20,321 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:20,321 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:20,321 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:20,321 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40077, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:20,322 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40077, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:20,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=145 2024-12-05T23:20:20,424 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40077, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:20,424 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=146}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x33d41122 to 127.0.0.1:50903 2024-12-05T23:20:20,424 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=146}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:20,447 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:20,447 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:20,447 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=146 2024-12-05T23:20:20,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=146 2024-12-05T23:20:20,450 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:20,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=146, resume processing ppid=145 2024-12-05T23:20:20,451 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-05T23:20:20,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=146, ppid=145, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 283 msec 2024-12-05T23:20:20,452 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=145, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 424 msec 2024-12-05T23:20:20,477 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5a65eade to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@28b7a69c 2024-12-05T23:20:20,482 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:20,482 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:20,483 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@4da3e7c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:20,483 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:20,486 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:20,487 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:20,487 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca4007a connected 2024-12-05T23:20:20,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=145 2024-12-05T23:20:20,656 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:20,656 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:20,657 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:20,657 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:20,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=147, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:20,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=148, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:20,658 DEBUG [PEWorker-5 {}] procedure.MasterProcedureScheduler(583): PEER '1', shared lock count=0, exclusively locked by procId=147 2024-12-05T23:20:20,658 DEBUG [PEWorker-5 {}] procedure2.ProcedureExecutor(1470): LOCK_EVENT_WAIT pid=148, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:20,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-12-05T23:20:20,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-12-05T23:20:20,660 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:20,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:20,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:20,663 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:20,664 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:20,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-12-05T23:20:20,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-12-05T23:20:20,815 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:20,816 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=149 2024-12-05T23:20:20,816 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-05T23:20:20,816 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-05T23:20:20,816 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-05T23:20:20,817 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:20,817 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca4007a, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:20,817 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca4007a, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:20,920 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca4007a, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:20,920 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=149}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5a65eade to 127.0.0.1:50903 2024-12-05T23:20:20,920 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=149}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:20,920 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-05T23:20:20,924 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=149}] zookeeper.ZKUtil(111): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:20,928 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:20,928 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:20,928 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:20,928 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=149}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:20,928 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-12-05T23:20:20,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=149 2024-12-05T23:20:20,930 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:20,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=149, resume processing ppid=147 2024-12-05T23:20:20,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=149, ppid=147, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 266 msec 2024-12-05T23:20:20,935 DEBUG [PEWorker-1 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:20,935 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-05T23:20:20,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=147, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 278 msec 2024-12-05T23:20:20,937 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:20,938 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=148, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:20,939 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=148, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=281 msec 2024-12-05T23:20:20,939 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-12-05T23:20:20,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-12-05T23:20:20,977 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:20,977 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-05T23:20:20,977 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:20:20,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=150, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:20,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=150 2024-12-05T23:20:20,979 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:20,980 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=150, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:20,981 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=150, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-05T23:20:20,981 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:21,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=150 2024-12-05T23:20:21,087 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:20:21,111 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[1] Thread=221 (was 219) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/java.lang.invoke.VarHandleGuards.guard_LI_L(VarHandleGuards.java:685) java.base@17.0.11/java.util.concurrent.ForkJoinPool$WorkQueue.getSlot(ForkJoinPool.java:900) java.base@17.0.11/java.util.concurrent.ForkJoinPool.scan(ForkJoinPool.java:1647) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1622) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) - Thread LEAK? -, OpenFileDescriptor=450 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=562 (was 576), ProcessCount=11 (was 11), AvailableMemoryMB=3369 (was 3750) 2024-12-05T23:20:21,126 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[1] Thread=221, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=562, ProcessCount=11, AvailableMemoryMB=3368 2024-12-05T23:20:21,128 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:20:21,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=151, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:20:21,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-12-05T23:20:21,147 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:21,148 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:21,156 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:21,157 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4007b connected 2024-12-05T23:20:21,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-12-05T23:20:21,260 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4007b, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:21,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:21,425 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:21,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=152 2024-12-05T23:20:21,426 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-05T23:20:21,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-12-05T23:20:21,452 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=152}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:21,477 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:21,497 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=152 2024-12-05T23:20:21,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=152 2024-12-05T23:20:21,499 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:21,502 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=152, resume processing ppid=151 2024-12-05T23:20:21,502 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=152, ppid=151, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 226 msec 2024-12-05T23:20:21,502 INFO [PEWorker-2 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-05T23:20:21,503 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=151, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 374 msec 2024-12-05T23:20:21,547 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x153419c3 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4605788f 2024-12-05T23:20:21,554 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:21,554 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:21,554 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3f15867a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:21,555 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:21,558 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:21,559 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca4007d connected 2024-12-05T23:20:21,559 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:21,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-12-05T23:20:21,756 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:21,756 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-05T23:20:21,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=153, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-12-05T23:20:21,779 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:21,779 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:21,788 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:21,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-12-05T23:20:21,888 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:21,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:22,047 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:22,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=154 2024-12-05T23:20:22,048 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:22,050 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:22,050 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:22,050 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:22,050 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca4007d, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:22,051 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca4007d, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:22,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-12-05T23:20:22,157 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca4007d, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:22,158 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=154}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x153419c3 to 127.0.0.1:50903 2024-12-05T23:20:22,158 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=154}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:22,191 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:22,191 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:22,200 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=154 2024-12-05T23:20:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=154 2024-12-05T23:20:22,202 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:22,204 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=154, resume processing ppid=153 2024-12-05T23:20:22,204 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=154, ppid=153, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 310 msec 2024-12-05T23:20:22,204 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-05T23:20:22,208 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=153, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 447 msec 2024-12-05T23:20:22,254 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6d48d82f to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3c0bbe29 2024-12-05T23:20:22,273 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:22,273 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:22,274 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@14aaabd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:22,276 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:22,287 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:22,288 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:22,289 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40080 connected 2024-12-05T23:20:22,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-12-05T23:20:22,396 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:22,397 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:22,398 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-05T23:20:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=155, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:22,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-12-05T23:20:22,427 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:22,428 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:22,432 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:22,437 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40081 connected 2024-12-05T23:20:22,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-12-05T23:20:22,544 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40081, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:22,560 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:22,712 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:22,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=156 2024-12-05T23:20:22,713 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=156}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:22,714 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=156}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:22,714 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=156}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:22,715 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:22,715 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40080, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:22,715 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40080, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:22,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-12-05T23:20:22,825 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40080, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:22,825 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=156}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6d48d82f to 127.0.0.1:50903 2024-12-05T23:20:22,826 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=156}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:22,849 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=156}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:22,850 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=156}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:22,852 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=156}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=156 2024-12-05T23:20:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=156 2024-12-05T23:20:22,855 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:22,856 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=156, resume processing ppid=155 2024-12-05T23:20:22,856 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=156, ppid=155, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 295 msec 2024-12-05T23:20:22,856 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-05T23:20:22,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=155, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 457 msec 2024-12-05T23:20:22,893 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1b0bc5ed to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@34d6efba 2024-12-05T23:20:22,904 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:22,905 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:22,905 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@1af924b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:22,906 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:22,928 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:22,928 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:22,937 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40083 connected 2024-12-05T23:20:22,968 INFO [master/9e5740704f6b:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-05T23:20:22,968 INFO [master/9e5740704f6b:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-05T23:20:23,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-12-05T23:20:23,027 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:23,027 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:23,028 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns1:testNamespacesAndTableCfsConfigConflict1=null},bandwidth=0,serial=false 2024-12-05T23:20:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=157, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:23,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-12-05T23:20:23,057 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:23,058 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:23,068 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:23,071 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40084 connected 2024-12-05T23:20:23,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-12-05T23:20:23,173 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40084, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:23,173 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:23,174 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=157, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:23,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=157, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=147 msec 2024-12-05T23:20:23,175 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:23,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-12-05T23:20:23,347 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config 2024-12-05T23:20:23,349 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:23,350 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-05T23:20:23,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=158, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:23,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=158 2024-12-05T23:20:23,377 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:23,377 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:23,388 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:23,396 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40085 connected 2024-12-05T23:20:23,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=158 2024-12-05T23:20:23,501 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40085, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:23,510 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:23,662 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:23,662 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=159 2024-12-05T23:20:23,663 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=159}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:23,664 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=159}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:23,664 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=159}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:23,664 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:23,664 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40083, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:23,665 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40083, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:23,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=158 2024-12-05T23:20:23,767 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40083, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:23,767 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=159}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1b0bc5ed to 127.0.0.1:50903 2024-12-05T23:20:23,767 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=159}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:23,789 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=159}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:23,789 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=159}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:23,798 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-05T23:20:23,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=159 2024-12-05T23:20:23,801 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:23,802 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=159, resume processing ppid=158 2024-12-05T23:20:23,803 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=159, ppid=158, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 291 msec 2024-12-05T23:20:23,803 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-05T23:20:23,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=158, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 453 msec 2024-12-05T23:20:23,867 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6c397e42 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3254e403 2024-12-05T23:20:23,913 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:23,914 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:23,914 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@5b2763f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:23,915 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:23,928 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:23,929 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40087 connected 2024-12-05T23:20:23,930 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:23,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=158 2024-12-05T23:20:23,981 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:23,982 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:23,983 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-05T23:20:23,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=160, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:23,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=160 2024-12-05T23:20:24,018 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:24,025 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:24,052 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:24,068 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40088 connected 2024-12-05T23:20:24,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=160 2024-12-05T23:20:24,173 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40088, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:24,173 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:24,174 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=160, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:24,175 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=160, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=191 msec 2024-12-05T23:20:24,175 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:24,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=160 2024-12-05T23:20:24,307 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config 2024-12-05T23:20:24,308 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:24,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=161, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:24,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=161 2024-12-05T23:20:24,311 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:24,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:24,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:24,313 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:24,315 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:24,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=161 2024-12-05T23:20:24,467 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:24,468 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=162 2024-12-05T23:20:24,468 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=162}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-05T23:20:24,468 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=162}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-05T23:20:24,468 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=162}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-05T23:20:24,468 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:24,468 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40087, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:24,469 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40087, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:24,572 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40087, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:24,572 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=162}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6c397e42 to 127.0.0.1:50903 2024-12-05T23:20:24,572 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=162}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:24,572 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=162}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-05T23:20:24,574 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=162}] zookeeper.ZKUtil(111): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:24,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:24,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:24,578 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:24,578 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=162}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:24,578 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=162}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=162 2024-12-05T23:20:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=162 2024-12-05T23:20:24,581 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:24,582 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=162, resume processing ppid=161 2024-12-05T23:20:24,583 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=162, ppid=161, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 266 msec 2024-12-05T23:20:24,588 DEBUG [PEWorker-1 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:24,588 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-05T23:20:24,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=161, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 280 msec 2024-12-05T23:20:24,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=161 2024-12-05T23:20:24,628 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:24,629 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=163, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=163 2024-12-05T23:20:24,631 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:24,632 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=163, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:24,633 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=163, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-05T23:20:24,633 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:24,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=163 2024-12-05T23:20:24,739 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-05T23:20:24,741 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:20:24,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=164, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=164 2024-12-05T23:20:24,746 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:24,747 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=164, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:24,748 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=164, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec 2024-12-05T23:20:24,748 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=164 2024-12-05T23:20:24,847 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:20:24,862 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[1] Thread=220 (was 221), OpenFileDescriptor=446 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=562 (was 562), ProcessCount=11 (was 11), AvailableMemoryMB=3285 (was 3368) 2024-12-05T23:20:24,873 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAddRemovePeer[1] Thread=220, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=562, ProcessCount=11, AvailableMemoryMB=3285 2024-12-05T23:20:24,874 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:20:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=165, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:20:24,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-12-05T23:20:24,893 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:24,894 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:24,908 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:24,909 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40089 connected 2024-12-05T23:20:24,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-12-05T23:20:25,011 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40089, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:25,019 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:25,171 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:25,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=166 2024-12-05T23:20:25,172 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=166}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-05T23:20:25,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-12-05T23:20:25,204 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=166}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:25,222 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=166}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:25,241 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=166}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=166 2024-12-05T23:20:25,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=166 2024-12-05T23:20:25,244 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:25,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=166, resume processing ppid=165 2024-12-05T23:20:25,247 INFO [PEWorker-5 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-05T23:20:25,247 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=166, ppid=165, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 225 msec 2024-12-05T23:20:25,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=165, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 373 msec 2024-12-05T23:20:25,278 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2ed1cc1a to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@216e07f7 2024-12-05T23:20:25,282 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:25,282 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:25,282 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@579aac5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:25,283 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:25,285 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:25,285 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:25,288 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca4008b connected 2024-12-05T23:20:25,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-12-05T23:20:25,507 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:25,507 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:20:25,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=167, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:20:25,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=167 2024-12-05T23:20:25,532 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:25,533 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:25,536 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:25,540 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4008c connected 2024-12-05T23:20:25,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=167 2024-12-05T23:20:25,644 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:121) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:25,645 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4008c, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:25,645 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=167, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:121) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:25,646 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=167, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=138 msec 2024-12-05T23:20:25,646 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:25,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=167 2024-12-05T23:20:25,826 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Replication peer 1 already exists 2024-12-05T23:20:25,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-05T23:20:25,829 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:20:25,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=168, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:25,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=168 2024-12-05T23:20:25,834 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:25,836 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=168, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:25,837 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=168, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=7 msec 2024-12-05T23:20:25,837 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:25,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=168 2024-12-05T23:20:25,937 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:20:25,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-05T23:20:25,938 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=2, config=clusterKey=127.0.0.1:50903:/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:20:25,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=169, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:20:25,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=169 2024-12-05T23:20:25,961 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:25,961 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:25,969 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:25,969 ERROR [PEWorker-2-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$360/0x00007f9a60914e50@7eb4a271 rejected from java.util.concurrent.ThreadPoolExecutor@707343d7[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 0] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.PendingWatcher.process(PendingWatcher.java:38) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-12-05T23:20:25,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-12-05T23:20:26,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=169 2024-12-05T23:20:26,070 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:26,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:26,230 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:26,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=170 2024-12-05T23:20:26,231 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=2, type=ADD_PEER 2024-12-05T23:20:26,250 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=170}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:26,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=169 2024-12-05T23:20:26,266 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.ReplicationSource(229): queueId=2, ReplicationSource: 2, currentBandwidth=0 2024-12-05T23:20:26,270 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=170 2024-12-05T23:20:26,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=170 2024-12-05T23:20:26,272 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 2 for ADD on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:26,277 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=170, resume processing ppid=169 2024-12-05T23:20:26,277 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=170, ppid=169, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 193 msec 2024-12-05T23:20:26,277 INFO [PEWorker-4 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 2, config clusterKey=127.0.0.1:50903:/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-05T23:20:26,280 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=169, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 340 msec 2024-12-05T23:20:26,286 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 2000 millis 2024-12-05T23:20:26,340 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3feb72eb to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@764e8b4a 2024-12-05T23:20:26,354 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,2 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:26,355 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,2 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:26,355 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@2f6548d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:26,356 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 2 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:26,360 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,2-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 20x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:26,360 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,2 {}] regionserver.ReplicationSource(569): peerId=2, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:26,368 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 2-0x10184ed0ca4008f connected 2024-12-05T23:20:26,434 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T23:20:26,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=169 2024-12-05T23:20:26,566 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 2 completed 2024-12-05T23:20:26,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-05T23:20:26,569 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:26,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=171, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:26,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=171 2024-12-05T23:20:26,575 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:26,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:26,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:26,579 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:26,580 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=172, ppid=171, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:26,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=171 2024-12-05T23:20:26,732 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:26,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=172 2024-12-05T23:20:26,733 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-05T23:20:26,733 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-05T23:20:26,733 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-05T23:20:26,733 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:26,733 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca4008b, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:26,734 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca4008b, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:26,836 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca4008b, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:26,837 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=172}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2ed1cc1a to 127.0.0.1:50903 2024-12-05T23:20:26,837 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=172}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:26,837 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-05T23:20:26,845 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=172}] zookeeper.ZKUtil(111): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:26,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:26,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:26,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:26,848 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=172}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:26,848 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=172 2024-12-05T23:20:26,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=172 2024-12-05T23:20:26,851 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:26,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=172, resume processing ppid=171 2024-12-05T23:20:26,853 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=172, ppid=171, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 271 msec 2024-12-05T23:20:26,857 DEBUG [PEWorker-1 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:26,857 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-05T23:20:26,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=171, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 288 msec 2024-12-05T23:20:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=171 2024-12-05T23:20:26,887 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-05T23:20:26,887 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:20:26,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=173, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:26,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=173 2024-12-05T23:20:26,890 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/peer-state 2024-12-05T23:20:26,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/peer-state 2024-12-05T23:20:26,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/2 2024-12-05T23:20:26,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2 2024-12-05T23:20:26,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=174, ppid=173, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:26,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=173 2024-12-05T23:20:27,046 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:27,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=174 2024-12-05T23:20:27,047 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=174}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=2, type=REMOVE_PEER 2024-12-05T23:20:27,047 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=174}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 2: 0 2024-12-05T23:20:27,047 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=174}] regionserver.ReplicationSource(657): peerId=2, Closing source 2 because: Replication stream was removed by a user 2024-12-05T23:20:27,047 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,2 {}] regionserver.ReplicationSource(508): peerId=2, Interrupted while sleeping between retries 2024-12-05T23:20:27,047 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,2 {}] zookeeper.ZKUtil(165): connection to cluster: 2-0x10184ed0ca4008f, quorum=127.0.0.1:50903, baseZNode=/hbase-test2 Unable to set watcher on znode (/hbase-test2/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:27,047 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,2 {}] zookeeper.ZKWatcher(734): connection to cluster: 2-0x10184ed0ca4008f, quorum=127.0.0.1:50903, baseZNode=/hbase-test2 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:27,149 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,2-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 2-0x10184ed0ca4008f, quorum=127.0.0.1:50903, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:27,150 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=174}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3feb72eb to 127.0.0.1:50903 2024-12-05T23:20:27,150 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=174}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:27,150 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=174}] regionserver.ReplicationSourceManager(430): Done with the queue 2 2024-12-05T23:20:27,151 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=174}] zookeeper.ZKUtil(111): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/2/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:27,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/2/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:27,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/2 2024-12-05T23:20:27,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/2 2024-12-05T23:20:27,154 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=174}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/2 not found in hfile reference queue. 2024-12-05T23:20:27,154 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=174}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=174 2024-12-05T23:20:27,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=174 2024-12-05T23:20:27,157 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 2 for REMOVE on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:27,159 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=174, resume processing ppid=173 2024-12-05T23:20:27,159 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=174, ppid=173, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 263 msec 2024-12-05T23:20:27,163 DEBUG [PEWorker-5 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/2 not found in hfile reference queue. 2024-12-05T23:20:27,163 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(76): Successfully removed peer 2 2024-12-05T23:20:27,164 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=173, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 275 msec 2024-12-05T23:20:27,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=173 2024-12-05T23:20:27,206 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 2 completed 2024-12-05T23:20:27,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-05T23:20:27,207 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:27,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=175, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:27,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=175 2024-12-05T23:20:27,209 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:27,210 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=175, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:27,210 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=175, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-05T23:20:27,210 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:27,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=175 2024-12-05T23:20:27,317 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-05T23:20:27,317 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:20:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=176, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:27,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=176 2024-12-05T23:20:27,319 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:27,323 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=176, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:27,324 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=176, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec 2024-12-05T23:20:27,324 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:27,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=176 2024-12-05T23:20:27,427 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:20:27,443 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAddRemovePeer[1] Thread=220 (was 220), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=581 (was 562) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=3139 (was 3285) 2024-12-05T23:20:27,455 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[1] Thread=220, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=581, ProcessCount=11, AvailableMemoryMB=3138 2024-12-05T23:20:27,456 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=whatever,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:20:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=177, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:20:27,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=177 2024-12-05T23:20:27,458 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:291) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:288) ~[classes/:?] ... 12 more 2024-12-05T23:20:27,459 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=177, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:291) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:288) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:27,460 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=177, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=4 msec 2024-12-05T23:20:27,460 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:27,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=177 2024-12-05T23:20:27,566 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Can not instantiate configured replication endpoint class=whatever 2024-12-05T23:20:27,567 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=178, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:27,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=178 2024-12-05T23:20:27,569 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:27,570 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=178, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:27,571 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=178, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-05T23:20:27,571 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:27,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=178 2024-12-05T23:20:27,676 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-05T23:20:27,677 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:20:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=179, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=179 2024-12-05T23:20:27,679 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:27,680 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=179, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:27,681 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=179, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-05T23:20:27,681 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:27,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=179 2024-12-05T23:20:27,786 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:20:27,800 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[1] Thread=220 (was 220), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=581 (was 581), ProcessCount=11 (was 11), AvailableMemoryMB=3137 (was 3138) 2024-12-05T23:20:27,811 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[1] Thread=220, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=581, ProcessCount=11, AvailableMemoryMB=3136 2024-12-05T23:20:27,811 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=whatever,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:20:27,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=180, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:20:27,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=180 2024-12-05T23:20:27,813 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] ... 13 more 2024-12-05T23:20:27,814 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=180, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:27,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=180, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=4 msec 2024-12-05T23:20:27,816 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:27,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=180 2024-12-05T23:20:27,917 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Invalid cluster key: whatever 2024-12-05T23:20:27,929 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:27,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=181, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:27,931 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:27,932 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=181, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:27,933 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=181, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-05T23:20:27,933 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:27,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=181 2024-12-05T23:20:27,937 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-05T23:20:27,938 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:20:27,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=182, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:27,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=182 2024-12-05T23:20:27,941 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:27,942 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=182, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:27,943 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=182, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-05T23:20:27,943 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:28,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=182 2024-12-05T23:20:28,047 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:20:28,072 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[1] Thread=220 (was 220), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=581 (was 581), ProcessCount=11 (was 11), AvailableMemoryMB=3127 (was 3136) 2024-12-05T23:20:28,099 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[1] Thread=220, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=581, ProcessCount=11, AvailableMemoryMB=3127 2024-12-05T23:20:28,100 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:20:28,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=183, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:20:28,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=183 2024-12-05T23:20:28,121 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:28,121 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:28,125 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:28,125 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40090 connected 2024-12-05T23:20:28,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=183 2024-12-05T23:20:28,227 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40090, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:28,239 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=184, ppid=183, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:28,392 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:28,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=184 2024-12-05T23:20:28,393 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-05T23:20:28,415 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=184}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:28,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=183 2024-12-05T23:20:28,442 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:28,458 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=184 2024-12-05T23:20:28,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=184 2024-12-05T23:20:28,468 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:28,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=184, resume processing ppid=183 2024-12-05T23:20:28,469 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=184, ppid=183, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 229 msec 2024-12-05T23:20:28,469 INFO [PEWorker-1 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-05T23:20:28,471 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=183, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 370 msec 2024-12-05T23:20:28,500 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x13192735 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2e1cbd3a 2024-12-05T23:20:28,504 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:28,504 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:28,505 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@621c35f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:28,505 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:28,508 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:28,508 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:28,508 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40092 connected 2024-12-05T23:20:28,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=183 2024-12-05T23:20:28,727 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:28,727 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-05T23:20:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=185, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:28,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=185 2024-12-05T23:20:28,752 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:28,752 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:28,755 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:28,756 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40093 connected 2024-12-05T23:20:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=185 2024-12-05T23:20:28,858 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40093, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:28,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=186, ppid=185, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:29,016 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:29,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=186 2024-12-05T23:20:29,018 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:29,019 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:29,019 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:29,019 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:29,020 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40092, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:29,021 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40092, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=185 2024-12-05T23:20:29,125 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40092, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:29,125 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=186}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x13192735 to 127.0.0.1:50903 2024-12-05T23:20:29,125 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=186}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:29,148 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:29,148 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:29,149 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=186 2024-12-05T23:20:29,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=186 2024-12-05T23:20:29,151 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:29,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=186, resume processing ppid=185 2024-12-05T23:20:29,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=186, ppid=185, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 288 msec 2024-12-05T23:20:29,157 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-05T23:20:29,160 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=185, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 430 msec 2024-12-05T23:20:29,185 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3f4b6905 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3cb67004 2024-12-05T23:20:29,202 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:29,202 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:29,202 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@70b5e868, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:29,203 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:29,208 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:29,209 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:29,209 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40095 connected 2024-12-05T23:20:29,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=185 2024-12-05T23:20:29,357 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:29,360 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:29,361 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-12-05T23:20:29,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=187, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:29,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=187 2024-12-05T23:20:29,385 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:29,386 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:29,390 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:29,391 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40096 connected 2024-12-05T23:20:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=187 2024-12-05T23:20:29,493 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40096, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:29,496 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=188, ppid=187, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:29,648 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:29,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=188 2024-12-05T23:20:29,649 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:29,650 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:29,650 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:29,650 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:29,650 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40095, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:29,651 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40095, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:29,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=187 2024-12-05T23:20:29,767 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40095, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:29,767 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=188}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3f4b6905 to 127.0.0.1:50903 2024-12-05T23:20:29,767 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=188}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:29,792 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:29,792 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:29,792 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=188 2024-12-05T23:20:29,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=188 2024-12-05T23:20:29,794 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:29,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=188, resume processing ppid=187 2024-12-05T23:20:29,799 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=188, ppid=187, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 298 msec 2024-12-05T23:20:29,799 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-12-05T23:20:29,801 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=187, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 438 msec 2024-12-05T23:20:29,841 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x72211398 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@675b8cec 2024-12-05T23:20:29,844 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:29,844 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:29,844 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@77b1aa89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:29,845 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:29,848 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:29,849 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:29,849 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca40098 connected 2024-12-05T23:20:29,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=187 2024-12-05T23:20:29,997 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:29,997 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:29,998 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:29,998 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-05T23:20:29,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=189, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:30,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=189 2024-12-05T23:20:30,023 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:30,024 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:30,028 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:30,029 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40099 connected 2024-12-05T23:20:30,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=189 2024-12-05T23:20:30,130 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca40099, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:30,134 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=190, ppid=189, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:30,289 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:30,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=190 2024-12-05T23:20:30,290 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:30,291 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:30,291 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:30,291 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:30,292 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca40098, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:30,292 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca40098, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:30,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=189 2024-12-05T23:20:30,394 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca40098, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:30,396 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=190}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x72211398 to 127.0.0.1:50903 2024-12-05T23:20:30,396 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=190}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:30,419 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:30,419 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:30,419 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=190 2024-12-05T23:20:30,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=190 2024-12-05T23:20:30,422 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:30,423 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=190, resume processing ppid=189 2024-12-05T23:20:30,424 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=190, ppid=189, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 288 msec 2024-12-05T23:20:30,424 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-05T23:20:30,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=189, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 426 msec 2024-12-05T23:20:30,462 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x617dd284 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@33fcf188 2024-12-05T23:20:30,481 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:30,481 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:30,482 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@a9783a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:30,483 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:30,487 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:30,488 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:30,490 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca4009b connected 2024-12-05T23:20:30,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=189 2024-12-05T23:20:30,626 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:30,627 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:30,627 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:30,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=191, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:30,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=191 2024-12-05T23:20:30,631 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:30,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:30,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:30,633 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:30,634 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=192, ppid=191, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:30,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=191 2024-12-05T23:20:30,785 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:30,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=192 2024-12-05T23:20:30,786 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=192}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-05T23:20:30,786 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=192}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-05T23:20:30,786 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=192}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-05T23:20:30,786 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:30,786 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca4009b, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:30,786 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca4009b, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:30,891 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca4009b, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:30,892 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=192}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x617dd284 to 127.0.0.1:50903 2024-12-05T23:20:30,892 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=192}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:30,892 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=192}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-05T23:20:30,894 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=192}] zookeeper.ZKUtil(111): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:30,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:30,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:30,897 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:30,897 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=192}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:30,897 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=192}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=192 2024-12-05T23:20:30,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=192 2024-12-05T23:20:30,899 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:30,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=192, resume processing ppid=191 2024-12-05T23:20:30,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=192, ppid=191, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 267 msec 2024-12-05T23:20:30,904 DEBUG [PEWorker-4 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:30,905 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-05T23:20:30,906 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=191, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 278 msec 2024-12-05T23:20:30,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=191 2024-12-05T23:20:30,947 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:30,947 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:30,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=193, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:30,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=193 2024-12-05T23:20:30,953 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:30,954 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=193, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:30,955 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=193, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=7 msec 2024-12-05T23:20:30,955 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:31,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=193 2024-12-05T23:20:31,057 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-05T23:20:31,057 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:20:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=194, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:31,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=194 2024-12-05T23:20:31,059 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:31,062 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=194, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:31,063 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=194, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-05T23:20:31,063 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:31,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=194 2024-12-05T23:20:31,167 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:20:31,183 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[1] Thread=220 (was 220), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=574 (was 581), ProcessCount=11 (was 11), AvailableMemoryMB=3058 (was 3127) 2024-12-05T23:20:31,197 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[1] Thread=220, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=574, ProcessCount=11, AvailableMemoryMB=3057 2024-12-05T23:20:31,198 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:20:31,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=195, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:20:31,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=195 2024-12-05T23:20:31,226 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:31,227 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:31,231 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:31,232 ERROR [PEWorker-5-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$360/0x00007f9a60914e50@12963b83 rejected from java.util.concurrent.ThreadPoolExecutor@16a9ad09[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 0] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.PendingWatcher.process(PendingWatcher.java:38) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-12-05T23:20:31,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=195 2024-12-05T23:20:31,334 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:31,342 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=196, ppid=195, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:31,494 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:31,495 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=196 2024-12-05T23:20:31,496 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-05T23:20:31,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=195 2024-12-05T23:20:31,541 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=196}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:31,564 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:31,597 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=196 2024-12-05T23:20:31,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=196 2024-12-05T23:20:31,600 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:31,602 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=196, resume processing ppid=195 2024-12-05T23:20:31,602 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=196, ppid=195, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 258 msec 2024-12-05T23:20:31,602 INFO [PEWorker-4 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-05T23:20:31,604 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=195, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 404 msec 2024-12-05T23:20:31,645 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4bfecd32 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@87f58b6 2024-12-05T23:20:31,677 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:31,678 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:31,678 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@b6e9452, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:31,679 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:31,685 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:31,685 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:31,691 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca4009e connected 2024-12-05T23:20:31,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=195 2024-12-05T23:20:31,837 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:31,837 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-05T23:20:31,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=197, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:31,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=197 2024-12-05T23:20:31,858 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:31,859 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:31,863 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:31,872 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4009f connected 2024-12-05T23:20:31,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=197 2024-12-05T23:20:31,974 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca4009f, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:31,978 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=198, ppid=197, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:32,129 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:32,130 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=198 2024-12-05T23:20:32,130 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:32,131 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:32,131 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:32,131 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:32,132 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca4009e, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:32,132 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca4009e, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:32,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=197 2024-12-05T23:20:32,234 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca4009e, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:32,234 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=198}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4bfecd32 to 127.0.0.1:50903 2024-12-05T23:20:32,234 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=198}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:32,260 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:32,262 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:32,270 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=198 2024-12-05T23:20:32,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=198 2024-12-05T23:20:32,288 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:32,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=198, resume processing ppid=197 2024-12-05T23:20:32,300 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=198, ppid=197, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 310 msec 2024-12-05T23:20:32,300 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-05T23:20:32,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=197, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 463 msec 2024-12-05T23:20:32,317 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6c6a7ea4 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2e652d1c 2024-12-05T23:20:32,339 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T23:20:32,349 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:32,350 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:32,350 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@55f99d7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:32,352 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:32,383 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:32,389 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca400a1 connected 2024-12-05T23:20:32,393 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:32,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=197 2024-12-05T23:20:32,470 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:32,470 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:32,471 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-12-05T23:20:32,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=199, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:32,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=199 2024-12-05T23:20:32,523 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:32,523 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:32,526 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:32,532 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca400a2 connected 2024-12-05T23:20:32,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=199 2024-12-05T23:20:32,635 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca400a2, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:32,639 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=200, ppid=199, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:32,792 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:32,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=200 2024-12-05T23:20:32,793 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:32,794 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:32,794 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:32,795 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:32,795 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca400a1, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:32,795 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca400a1, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:32,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=199 2024-12-05T23:20:32,899 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=200}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6c6a7ea4 to 127.0.0.1:50903 2024-12-05T23:20:32,899 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=200}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:32,900 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca400a1, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:32,952 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:32,952 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:32,956 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=200 2024-12-05T23:20:32,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=200 2024-12-05T23:20:32,966 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:32,971 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=200, resume processing ppid=199 2024-12-05T23:20:32,971 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=200, ppid=199, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 327 msec 2024-12-05T23:20:32,971 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-12-05T23:20:32,973 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=199, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 500 msec 2024-12-05T23:20:33,004 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5fde4f0d to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@45ddce26 2024-12-05T23:20:33,017 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:33,017 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:33,018 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@1ba2a32f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:33,019 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:33,022 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:33,022 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:33,024 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca400a4 connected 2024-12-05T23:20:33,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=199 2024-12-05T23:20:33,116 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:33,124 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:33,136 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:33,140 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-05T23:20:33,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=201, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:33,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=201 2024-12-05T23:20:33,172 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:33,173 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:33,183 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:33,197 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca400a5 connected 2024-12-05T23:20:33,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=201 2024-12-05T23:20:33,302 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca400a5, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:33,309 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=202, ppid=201, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:33,461 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:33,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=202 2024-12-05T23:20:33,462 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:33,465 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:33,465 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:33,465 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:33,465 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca400a4, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:33,465 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca400a4, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:33,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=201 2024-12-05T23:20:33,569 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca400a4, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:33,569 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=202}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5fde4f0d to 127.0.0.1:50903 2024-12-05T23:20:33,569 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=202}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:33,588 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:33,589 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:33,589 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=202 2024-12-05T23:20:33,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=202 2024-12-05T23:20:33,609 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:33,611 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=202, resume processing ppid=201 2024-12-05T23:20:33,611 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=202, ppid=201, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 300 msec 2024-12-05T23:20:33,611 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-05T23:20:33,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=201, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 471 msec 2024-12-05T23:20:33,634 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3ea70b39 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@700fd47a 2024-12-05T23:20:33,664 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:33,665 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:33,665 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@62b5632b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:33,667 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:33,688 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:33,696 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:33,728 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca400a7 connected 2024-12-05T23:20:33,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=201 2024-12-05T23:20:33,777 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:33,777 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:33,778 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:33,779 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-05T23:20:33,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=203, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:33,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=203 2024-12-05T23:20:33,801 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:33,802 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:33,811 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:33,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=203 2024-12-05T23:20:33,910 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:33,913 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=204, ppid=203, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:34,065 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:34,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=204 2024-12-05T23:20:34,065 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:34,066 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:34,067 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:34,067 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:34,067 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca400a7, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:34,067 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca400a7, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:34,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=203 2024-12-05T23:20:34,170 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca400a7, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:34,170 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=204}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3ea70b39 to 127.0.0.1:50903 2024-12-05T23:20:34,170 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=204}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:34,189 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:34,189 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:34,189 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=204 2024-12-05T23:20:34,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=204 2024-12-05T23:20:34,191 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:34,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=204, resume processing ppid=203 2024-12-05T23:20:34,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=204, ppid=203, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 278 msec 2024-12-05T23:20:34,193 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-05T23:20:34,194 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=203, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 414 msec 2024-12-05T23:20:34,223 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2a53d388 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@c571b2b 2024-12-05T23:20:34,227 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:34,227 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:34,228 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3f1b608b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:34,228 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:34,231 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:34,232 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:34,241 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca400aa connected 2024-12-05T23:20:34,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=203 2024-12-05T23:20:34,407 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:34,407 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:34,408 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:34,409 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-05T23:20:34,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=205, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:34,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=205 2024-12-05T23:20:34,427 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:34,428 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:34,432 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:34,432 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca400ab connected 2024-12-05T23:20:34,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=205 2024-12-05T23:20:34,530 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-05T23:20:34,534 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca400ab, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:34,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=206, ppid=205, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:34,690 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:34,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=206 2024-12-05T23:20:34,691 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:34,692 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:34,692 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:34,692 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:34,692 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca400aa, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:34,693 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca400aa, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:34,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=205 2024-12-05T23:20:34,794 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca400aa, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:34,795 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=206}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2a53d388 to 127.0.0.1:50903 2024-12-05T23:20:34,795 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=206}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:34,817 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:34,817 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:34,818 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=206 2024-12-05T23:20:34,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=206 2024-12-05T23:20:34,821 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:34,823 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=206, resume processing ppid=205 2024-12-05T23:20:34,823 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=206, ppid=205, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 283 msec 2024-12-05T23:20:34,823 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-05T23:20:34,824 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=205, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 414 msec 2024-12-05T23:20:34,861 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x50ad1066 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5cbc027f 2024-12-05T23:20:34,877 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:34,878 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:34,878 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@52c93a44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:34,880 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:34,889 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:34,889 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca400ad connected 2024-12-05T23:20:34,896 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:35,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=205 2024-12-05T23:20:35,037 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:35,037 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:35,038 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:35,039 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-05T23:20:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=207, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:35,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=207 2024-12-05T23:20:35,063 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:35,063 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:35,067 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:35,068 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca400ae connected 2024-12-05T23:20:35,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=207 2024-12-05T23:20:35,177 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca400ae, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:35,181 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=208, ppid=207, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:35,332 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:35,333 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=208 2024-12-05T23:20:35,333 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:35,335 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:35,335 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:35,335 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:35,335 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca400ad, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:35,335 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca400ad, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:35,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=207 2024-12-05T23:20:35,437 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca400ad, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:35,438 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=208}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x50ad1066 to 127.0.0.1:50903 2024-12-05T23:20:35,438 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=208}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:35,468 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:35,469 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:35,469 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=208 2024-12-05T23:20:35,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=208 2024-12-05T23:20:35,471 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:35,473 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=208, resume processing ppid=207 2024-12-05T23:20:35,473 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=208, ppid=207, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 290 msec 2024-12-05T23:20:35,473 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-05T23:20:35,475 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=207, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 434 msec 2024-12-05T23:20:35,513 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x57a27316 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7fbd6d7c 2024-12-05T23:20:35,520 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:35,520 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:35,520 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@1ed0e132, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:35,521 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:35,527 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:35,528 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:35,528 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca400b0 connected 2024-12-05T23:20:35,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=207 2024-12-05T23:20:35,667 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:35,667 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:35,668 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-05T23:20:35,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=209, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:35,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=209 2024-12-05T23:20:35,692 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:35,692 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:35,695 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:35,695 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca400b1 connected 2024-12-05T23:20:35,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=209 2024-12-05T23:20:35,797 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca400b1, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:35,802 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=210, ppid=209, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:35,954 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:35,954 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=210 2024-12-05T23:20:35,954 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:35,956 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=210 2024-12-05T23:20:35,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=210 2024-12-05T23:20:35,958 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:35,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=210, resume processing ppid=209 2024-12-05T23:20:35,960 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=210, ppid=209, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 156 msec 2024-12-05T23:20:35,960 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-05T23:20:35,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=209, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 292 msec 2024-12-05T23:20:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=209 2024-12-05T23:20:35,986 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:35,986 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:35,987 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:35,988 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-12-05T23:20:35,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=211, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:35,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=211 2024-12-05T23:20:36,007 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:36,007 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:36,010 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:36,010 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca400b2 connected 2024-12-05T23:20:36,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-12-05T23:20:36,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=211 2024-12-05T23:20:36,113 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca400b2, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:36,122 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=212, ppid=211, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:36,273 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:36,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=212 2024-12-05T23:20:36,274 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:36,275 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:36,275 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:36,275 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:36,275 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca400b0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:36,275 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca400b0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:36,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=211 2024-12-05T23:20:36,378 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca400b0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:36,378 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=212}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x57a27316 to 127.0.0.1:50903 2024-12-05T23:20:36,378 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=212}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:36,397 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:36,397 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:36,398 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=212 2024-12-05T23:20:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=212 2024-12-05T23:20:36,400 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:36,402 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=212, resume processing ppid=211 2024-12-05T23:20:36,402 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=212, ppid=211, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 278 msec 2024-12-05T23:20:36,402 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-12-05T23:20:36,403 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=211, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 414 msec 2024-12-05T23:20:36,439 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x68ea54e5 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4c00c683 2024-12-05T23:20:36,444 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:36,445 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:36,445 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@4b728a2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:36,445 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:36,448 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:36,448 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:36,448 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca400b4 connected 2024-12-05T23:20:36,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=211 2024-12-05T23:20:36,617 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:36,617 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:36,618 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-12-05T23:20:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=213, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:36,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=213 2024-12-05T23:20:36,641 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:36,641 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:36,644 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:36,644 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca400b5 connected 2024-12-05T23:20:36,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=213 2024-12-05T23:20:36,746 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca400b5, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:36,751 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=214, ppid=213, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:36,903 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:36,903 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=214 2024-12-05T23:20:36,903 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:36,905 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:36,905 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:36,905 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:36,905 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca400b4, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:36,905 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca400b4, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:36,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=213 2024-12-05T23:20:37,007 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca400b4, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:37,007 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=214}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x68ea54e5 to 127.0.0.1:50903 2024-12-05T23:20:37,007 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=214}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:37,030 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:37,031 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:37,031 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=214 2024-12-05T23:20:37,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=214 2024-12-05T23:20:37,033 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:37,035 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=214, resume processing ppid=213 2024-12-05T23:20:37,035 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=214, ppid=213, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 282 msec 2024-12-05T23:20:37,035 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-12-05T23:20:37,038 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=213, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 417 msec 2024-12-05T23:20:37,074 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x14bd7838 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6f505b2 2024-12-05T23:20:37,079 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:37,079 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:37,079 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@7c1ab9a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:37,080 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:37,085 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:37,085 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:37,085 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca400b7 connected 2024-12-05T23:20:37,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=213 2024-12-05T23:20:37,247 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:37,247 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:37,251 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:37,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=215, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:37,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=215 2024-12-05T23:20:37,255 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:37,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:37,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:37,258 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:37,259 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=216, ppid=215, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:37,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=215 2024-12-05T23:20:37,411 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:37,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=216 2024-12-05T23:20:37,412 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=216}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-05T23:20:37,412 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=216}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-05T23:20:37,412 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=216}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-05T23:20:37,412 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:37,412 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca400b7, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:37,412 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca400b7, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:37,518 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=216}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x14bd7838 to 127.0.0.1:50903 2024-12-05T23:20:37,518 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=216}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:37,518 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=216}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-05T23:20:37,520 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca400b7, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:37,520 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=216}] zookeeper.ZKUtil(111): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:37,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:37,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:37,525 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:37,528 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=216}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:37,528 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=216}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=216 2024-12-05T23:20:37,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=216 2024-12-05T23:20:37,531 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:37,532 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=216, resume processing ppid=215 2024-12-05T23:20:37,532 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=216, ppid=215, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 272 msec 2024-12-05T23:20:37,536 DEBUG [PEWorker-5 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:37,536 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-05T23:20:37,537 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=215, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 285 msec 2024-12-05T23:20:37,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=215 2024-12-05T23:20:37,576 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:37,577 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:37,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=217, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:37,578 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:37,580 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=217, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:37,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=217 2024-12-05T23:20:37,581 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=217, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-05T23:20:37,581 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:37,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=217 2024-12-05T23:20:37,686 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-05T23:20:37,687 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:20:37,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=218, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:37,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=218 2024-12-05T23:20:37,688 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:37,689 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=218, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:37,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=218, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-05T23:20:37,690 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:37,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=218 2024-12-05T23:20:37,796 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:20:37,811 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[1] Thread=217 (was 220), OpenFileDescriptor=448 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=568 (was 574), ProcessCount=11 (was 11), AvailableMemoryMB=2917 (was 3057) 2024-12-05T23:20:37,821 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[1] Thread=217, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=568, ProcessCount=11, AvailableMemoryMB=2917 2024-12-05T23:20:37,822 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:20:37,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=219, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:20:37,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=219 2024-12-05T23:20:37,845 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:37,845 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:37,848 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:37,848 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca400b8 connected 2024-12-05T23:20:37,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=219 2024-12-05T23:20:37,951 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca400b8, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:37,963 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=220, ppid=219, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:38,115 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:38,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=220 2024-12-05T23:20:38,116 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-05T23:20:38,134 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=220}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:38,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=219 2024-12-05T23:20:38,149 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:38,159 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=220 2024-12-05T23:20:38,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=220 2024-12-05T23:20:38,161 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:38,163 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=220, resume processing ppid=219 2024-12-05T23:20:38,163 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=220, ppid=219, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 198 msec 2024-12-05T23:20:38,163 INFO [PEWorker-5 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-05T23:20:38,165 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=219, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 341 msec 2024-12-05T23:20:38,192 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4b11d094 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@be8e032 2024-12-05T23:20:38,196 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:38,196 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:38,197 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@6aaf3e04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:38,197 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:38,213 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:38,213 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:38,223 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca400ba connected 2024-12-05T23:20:38,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=219 2024-12-05T23:20:38,447 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:38,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-05T23:20:38,448 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3938): Client=jenkins//172.17.0.2 disable replication peer, id=1 2024-12-05T23:20:38,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=221, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure 2024-12-05T23:20:38,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=222, ppid=221, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:38,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=221 2024-12-05T23:20:38,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=221 2024-12-05T23:20:38,605 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:38,606 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=222 2024-12-05T23:20:38,606 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=DISABLE_PEER 2024-12-05T23:20:38,607 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-05T23:20:38,607 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-05T23:20:38,607 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:38,607 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca400ba, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:38,607 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca400ba, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:38,710 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca400ba, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:38,710 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=222}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4b11d094 to 127.0.0.1:50903 2024-12-05T23:20:38,710 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=222}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:38,748 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:38,748 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-05T23:20:38,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=221 2024-12-05T23:20:38,768 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=222 2024-12-05T23:20:38,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=222 2024-12-05T23:20:38,771 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for DISABLE on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:38,773 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=222, resume processing ppid=221 2024-12-05T23:20:38,773 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=222, ppid=221, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 318 msec 2024-12-05T23:20:38,773 INFO [PEWorker-4 {}] replication.DisablePeerProcedure(67): Successfully disabled peer 1 2024-12-05T23:20:38,774 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=221, state=SUCCESS; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure in 324 msec 2024-12-05T23:20:38,820 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6ec87dd0 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@77a93e17 2024-12-05T23:20:38,867 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:38,868 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:38,868 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3a69ae93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:38,869 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:38,880 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:38,882 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:38,888 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca400bc connected 2024-12-05T23:20:39,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=221 2024-12-05T23:20:39,077 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: DISABLE_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:39,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-05T23:20:39,078 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:39,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=223, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:39,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=223 2024-12-05T23:20:39,081 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:39,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:39,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:39,083 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:39,084 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=224, ppid=223, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:39,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=223 2024-12-05T23:20:39,235 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:39,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=224 2024-12-05T23:20:39,236 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=224}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-05T23:20:39,236 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=224}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-05T23:20:39,236 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=224}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-05T23:20:39,236 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:39,236 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca400bc, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:39,236 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca400bc, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:39,339 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca400bc, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:39,340 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=224}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6ec87dd0 to 127.0.0.1:50903 2024-12-05T23:20:39,340 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=224}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:39,340 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=224}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-05T23:20:39,342 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=224}] zookeeper.ZKUtil(111): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:39,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:39,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:39,344 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:39,345 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=224}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:39,345 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=224}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=224 2024-12-05T23:20:39,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=224 2024-12-05T23:20:39,347 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:39,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=224, resume processing ppid=223 2024-12-05T23:20:39,349 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=224, ppid=223, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 263 msec 2024-12-05T23:20:39,353 DEBUG [PEWorker-3 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:39,353 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-05T23:20:39,356 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=223, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 275 msec 2024-12-05T23:20:39,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=223 2024-12-05T23:20:39,396 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:39,397 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:39,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=225, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:39,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=225 2024-12-05T23:20:39,398 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:39,399 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=225, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:39,400 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=225, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-05T23:20:39,400 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:39,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=225 2024-12-05T23:20:39,506 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-05T23:20:39,508 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:20:39,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=226, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:39,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=226 2024-12-05T23:20:39,510 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:39,510 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=226, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:39,514 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=226, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-05T23:20:39,515 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:39,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=226 2024-12-05T23:20:39,629 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:20:39,659 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[1] Thread=217 (was 217), OpenFileDescriptor=448 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=568 (was 568), ProcessCount=11 (was 11), AvailableMemoryMB=2865 (was 2917) 2024-12-05T23:20:39,679 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[1] Thread=217, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=568, ProcessCount=11, AvailableMemoryMB=2864 2024-12-05T23:20:39,680 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=dummy_peer 2024-12-05T23:20:39,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=227, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:39,682 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer dummy_peer, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:39,683 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=227, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:39,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=227 2024-12-05T23:20:39,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=227, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-05T23:20:39,685 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:39,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=227 2024-12-05T23:20:39,797 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: dummy_peer failed with dummy_peer 2024-12-05T23:20:39,797 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:39,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=228, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:39,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=228 2024-12-05T23:20:39,799 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:39,800 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=228, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:39,802 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=228, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-05T23:20:39,802 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:39,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=228 2024-12-05T23:20:39,907 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-05T23:20:39,907 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:20:39,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=229, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:39,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=229 2024-12-05T23:20:39,909 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:39,910 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=229, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:39,911 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=229, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-05T23:20:39,911 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=229 2024-12-05T23:20:40,017 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:20:40,030 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[1] Thread=217 (was 217), OpenFileDescriptor=448 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=568 (was 568), ProcessCount=11 (was 11), AvailableMemoryMB=2863 (was 2864) 2024-12-05T23:20:40,040 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerConfig[1] Thread=217, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=568, ProcessCount=11, AvailableMemoryMB=2863 2024-12-05T23:20:40,041 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:20:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=230, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:20:40,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=230 2024-12-05T23:20:40,059 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:40,059 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:40,071 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:40,072 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca400bd connected 2024-12-05T23:20:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=230 2024-12-05T23:20:40,174 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca400bd, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:40,181 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=231, ppid=230, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:40,333 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:40,334 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=231 2024-12-05T23:20:40,334 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-05T23:20:40,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=230 2024-12-05T23:20:40,357 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=231}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:40,375 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:40,388 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=231 2024-12-05T23:20:40,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=231 2024-12-05T23:20:40,390 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:40,391 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=231, resume processing ppid=230 2024-12-05T23:20:40,391 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=231, ppid=230, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 209 msec 2024-12-05T23:20:40,391 INFO [PEWorker-1 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-05T23:20:40,393 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=230, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 351 msec 2024-12-05T23:20:40,422 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x31522e87 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@61910fec 2024-12-05T23:20:40,427 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:40,427 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:40,427 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@12e8cc0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:40,429 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:40,433 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:40,433 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca400bf connected 2024-12-05T23:20:40,434 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:40,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=230 2024-12-05T23:20:40,667 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:40,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-05T23:20:40,668 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:40,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=232, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:40,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=232 2024-12-05T23:20:40,671 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:40,674 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:40,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:40,675 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:40,675 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=233, ppid=232, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:40,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=232 2024-12-05T23:20:40,827 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:40,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=233 2024-12-05T23:20:40,828 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=233}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-05T23:20:40,828 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=233}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-05T23:20:40,828 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=233}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-05T23:20:40,828 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:40,828 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca400bf, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:40,828 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca400bf, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:40,930 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca400bf, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:40,930 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=233}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x31522e87 to 127.0.0.1:50903 2024-12-05T23:20:40,930 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=233}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:40,931 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=233}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-05T23:20:40,933 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=233}] zookeeper.ZKUtil(111): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:40,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:40,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:40,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:40,937 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=233}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:40,937 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=233}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=233 2024-12-05T23:20:40,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=233 2024-12-05T23:20:40,939 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:40,940 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=233, resume processing ppid=232 2024-12-05T23:20:40,940 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=233, ppid=232, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 264 msec 2024-12-05T23:20:40,942 DEBUG [PEWorker-5 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:40,942 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-05T23:20:40,946 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=232, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 275 msec 2024-12-05T23:20:40,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=232 2024-12-05T23:20:40,986 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:40,986 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:40,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=234, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:40,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=234 2024-12-05T23:20:40,988 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:40,989 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=234, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:40,990 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=234, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-05T23:20:40,990 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:41,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=234 2024-12-05T23:20:41,096 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-05T23:20:41,097 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:20:41,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=235, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:41,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=235 2024-12-05T23:20:41,098 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:41,099 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=235, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:41,100 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=235, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-05T23:20:41,100 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:41,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=235 2024-12-05T23:20:41,206 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:20:41,217 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerConfig[1] Thread=219 (was 217) - Thread LEAK? -, OpenFileDescriptor=450 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=579 (was 568) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2835 (was 2863) 2024-12-05T23:20:41,227 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[1] Thread=219, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=579, ProcessCount=11, AvailableMemoryMB=2834 2024-12-05T23:20:41,229 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:20:41,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-05T23:20:41,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] ipc.CallRunner(138): callId: 765 service: MasterService methodName: AddReplicationPeer size: 80 connection: 172.17.0.2:33460 deadline: 1733440901229, exception=java.io.IOException: Replication peer modification disabled 2024-12-05T23:20:41,230 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(168): Call to master failed, tries = 1, maxAttempts = 2, timeout = 120000 ms, time elapsed = 0 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:245) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:431) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 33 more 2024-12-05T23:20:41,336 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:20:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-05T23:20:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] ipc.CallRunner(138): callId: 766 service: MasterService methodName: AddReplicationPeer size: 80 connection: 172.17.0.2:33460 deadline: 1733440901335, exception=java.io.IOException: Replication peer modification disabled 2024-12-05T23:20:41,336 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(168): Call to master failed, tries = 2, maxAttempts = 2, timeout = 120000 ms, time elapsed = 107 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:245) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:431) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 33 more 2024-12-05T23:20:41,337 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Failed after attempts=2, exceptions: 2024-12-05T23:20:41.230Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) 2024-12-05T23:20:41.337Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) 2024-12-05T23:20:41,339 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:41,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=236, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:41,340 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:41,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=236 2024-12-05T23:20:41,341 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=236, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:41,341 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=236, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-05T23:20:41,342 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:41,385 DEBUG [master/9e5740704f6b:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 1588230740 changed from -1.0 to 0.0, refreshing cache 2024-12-05T23:20:41,390 DEBUG [master/9e5740704f6b:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 4cc297f79f625a9804bd9a56700799f5 changed from -1.0 to 0.0, refreshing cache 2024-12-05T23:20:41,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=236 2024-12-05T23:20:41,447 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-05T23:20:41,447 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:20:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=237, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=237 2024-12-05T23:20:41,448 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:41,449 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=237, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:41,450 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=237, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-05T23:20:41,450 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:41,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=237 2024-12-05T23:20:41,556 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:20:41,569 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[1] Thread=221 (was 219) Potentially hanging thread: region-location-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: region-location-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=450 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=579 (was 579), ProcessCount=11 (was 11), AvailableMemoryMB=2833 (was 2834) 2024-12-05T23:20:41,578 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[1] Thread=221, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=579, ProcessCount=11, AvailableMemoryMB=2832 2024-12-05T23:20:41,579 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:20:41,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=238, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:20:41,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=238 2024-12-05T23:20:41,594 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:41,594 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:41,599 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:41,600 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca400c0 connected 2024-12-05T23:20:41,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=238 2024-12-05T23:20:41,701 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca400c0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:41,706 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=239, ppid=238, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:41,857 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:41,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=239 2024-12-05T23:20:41,858 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-05T23:20:41,873 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=239}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:41,886 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:41,895 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=239 2024-12-05T23:20:41,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=238 2024-12-05T23:20:41,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=239 2024-12-05T23:20:41,897 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:41,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=239, resume processing ppid=238 2024-12-05T23:20:41,898 INFO [PEWorker-2 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-05T23:20:41,898 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=239, ppid=238, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 191 msec 2024-12-05T23:20:41,899 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=238, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 319 msec 2024-12-05T23:20:41,924 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7726b44d to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@75ca653a 2024-12-05T23:20:41,927 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:41,927 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:41,928 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@11e7abac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:41,928 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:41,930 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:41,931 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:41,931 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca400c2 connected 2024-12-05T23:20:42,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=238 2024-12-05T23:20:42,206 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:42,206 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:42,207 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:42,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=240, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:42,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=240 2024-12-05T23:20:42,209 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:42,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:42,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:42,211 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:42,212 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=241, ppid=240, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:42,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=240 2024-12-05T23:20:42,363 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:42,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=241 2024-12-05T23:20:42,364 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=241}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-05T23:20:42,364 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=241}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-05T23:20:42,364 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=241}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-05T23:20:42,364 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:42,364 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca400c2, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:42,364 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca400c2, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:42,469 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca400c2, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:42,469 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=241}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7726b44d to 127.0.0.1:50903 2024-12-05T23:20:42,469 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=241}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:42,469 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=241}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-05T23:20:42,471 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=241}] zookeeper.ZKUtil(111): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:42,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:42,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:42,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:42,473 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=241}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:42,473 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=241}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=241 2024-12-05T23:20:42,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=241 2024-12-05T23:20:42,475 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:42,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=241, resume processing ppid=240 2024-12-05T23:20:42,476 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=241, ppid=240, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 263 msec 2024-12-05T23:20:42,479 DEBUG [PEWorker-3 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:42,479 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-05T23:20:42,480 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=240, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 272 msec 2024-12-05T23:20:42,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=240 2024-12-05T23:20:42,526 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:42,527 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:20:42,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=242, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:42,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=242 2024-12-05T23:20:42,528 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:42,529 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=242, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:42,529 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=242, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-05T23:20:42,529 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:42,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=242 2024-12-05T23:20:42,636 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:20:42,648 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[1] Thread=220 (was 221), OpenFileDescriptor=450 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=579 (was 579), ProcessCount=11 (was 11), AvailableMemoryMB=2834 (was 2832) - AvailableMemoryMB LEAK? - 2024-12-05T23:20:42,658 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerBandwidth[1] Thread=220, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=579, ProcessCount=11, AvailableMemoryMB=2834 2024-12-05T23:20:42,658 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:20:42,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=243, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:20:42,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=243 2024-12-05T23:20:42,675 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:42,675 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:42,679 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:42,679 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca400c3 connected 2024-12-05T23:20:42,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=243 2024-12-05T23:20:42,781 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca400c3, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:42,790 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=244, ppid=243, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:42,941 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:42,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=244 2024-12-05T23:20:42,942 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=244}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-05T23:20:42,963 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=244}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:42,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=243 2024-12-05T23:20:42,980 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=244}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:42,989 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=244}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=244 2024-12-05T23:20:42,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=244 2024-12-05T23:20:42,991 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:42,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=244, resume processing ppid=243 2024-12-05T23:20:42,993 INFO [PEWorker-3 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-05T23:20:42,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=244, ppid=243, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 201 msec 2024-12-05T23:20:42,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=243, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 335 msec 2024-12-05T23:20:43,021 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0a956ad0 to 127.0.0.1:50903 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@548b14c 2024-12-05T23:20:43,026 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-05T23:20:43,027 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-05T23:20:43,027 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@39724654, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-05T23:20:43,028 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:43,031 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:43,032 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-05T23:20:43,032 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10184ed0ca400c5 connected 2024-12-05T23:20:43,037 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T23:20:43,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=243 2024-12-05T23:20:43,286 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:43,286 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:43,287 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-12-05T23:20:43,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=245, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-05T23:20:43,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=245 2024-12-05T23:20:43,302 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-05T23:20:43,302 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50903 2024-12-05T23:20:43,304 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id0x0, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-05T23:20:43,305 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca400c6 connected 2024-12-05T23:20:43,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=245 2024-12-05T23:20:43,406 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@914d3aecheck-peer-cluster-id-0x10184ed0ca400c6, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:43,411 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=246, ppid=245, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:43,562 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:43,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=246 2024-12-05T23:20:43,563 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-05T23:20:43,564 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=246 2024-12-05T23:20:43,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=246 2024-12-05T23:20:43,565 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:43,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=246, resume processing ppid=245 2024-12-05T23:20:43,566 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50903:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-12-05T23:20:43,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=246, ppid=245, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 154 msec 2024-12-05T23:20:43,567 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=245, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 279 msec 2024-12-05T23:20:43,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=245 2024-12-05T23:20:43,606 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-05T23:20:43,606 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-05T23:20:43,606 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:43,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=247, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:43,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=247 2024-12-05T23:20:43,608 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:43,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:43,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:43,610 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:43,610 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=248, ppid=247, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:43,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=247 2024-12-05T23:20:43,761 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:43,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=248 2024-12-05T23:20:43,762 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=248}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-05T23:20:43,762 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=248}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-05T23:20:43,762 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=248}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-05T23:20:43,762 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-05T23:20:43,762 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10184ed0ca400c5, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:43,763 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10184ed0ca400c5, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-05T23:20:43,870 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10184ed0ca400c5, quorum=127.0.0.1:50903, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:43,870 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=248}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0a956ad0 to 127.0.0.1:50903 2024-12-05T23:20:43,870 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=248}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:43,870 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=248}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-05T23:20:43,872 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=248}] zookeeper.ZKUtil(111): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:43,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:43,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:43,874 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:43,874 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=248}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:43,874 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=248}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=248 2024-12-05T23:20:43,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=248 2024-12-05T23:20:43,876 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:43,877 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=248, resume processing ppid=247 2024-12-05T23:20:43,878 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=248, ppid=247, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 266 msec 2024-12-05T23:20:43,880 DEBUG [PEWorker-5 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:43,880 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-05T23:20:43,881 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=247, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 274 msec 2024-12-05T23:20:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=247 2024-12-05T23:20:43,926 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:43,926 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:43,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=249, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:43,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=249 2024-12-05T23:20:43,928 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:43,929 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=249, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:43,930 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=249, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-05T23:20:43,930 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:44,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=249 2024-12-05T23:20:44,036 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-05T23:20:44,037 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:20:44,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=250, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=250 2024-12-05T23:20:44,038 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:44,039 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=250, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:44,039 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=250, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-05T23:20:44,039 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:44,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=250 2024-12-05T23:20:44,147 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:20:44,157 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerBandwidth[1] Thread=217 (was 220), OpenFileDescriptor=442 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=579 (was 579), ProcessCount=11 (was 11), AvailableMemoryMB=2794 (was 2834) 2024-12-05T23:20:44,165 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[1] Thread=217, OpenFileDescriptor=442, MaxFileDescriptor=1048576, SystemLoadAverage=579, ProcessCount=11, AvailableMemoryMB=2794 2024-12-05T23:20:44,166 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:20:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=251, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:20:44,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=251 2024-12-05T23:20:44,176 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=252, ppid=251, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:44,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=251 2024-12-05T23:20:44,327 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:44,328 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=252 2024-12-05T23:20:44,328 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=252}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-05T23:20:44,356 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=252}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-05T23:20:44,365 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=252}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=252 2024-12-05T23:20:44,365 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(587): peerId=1, queueId=1 (queues=1) is replicating from cluster=a6e07b25-fe39-42c3-aefd-b0129d0da2f3 to cluster=a6e07b25-fe39-42c3-aefd-b0129d0da2f3 2024-12-05T23:20:44,365 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSource(343): peerId=1, starting shipping worker for walGroupId=9e5740704f6b%2C36187%2C1733440777223 2024-12-05T23:20:44,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=252 2024-12-05T23:20:44,365 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1 {}] regionserver.ReplicationSourceWALReader(112): peerClusterZnode=1, ReplicationSourceWALReaderThread : 1 inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-12-05T23:20:44,365 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1.replicationSource.shipper9e5740704f6b%2C36187%2C1733440777223,1 {}] regionserver.ReplicationSourceShipper(98): Running ReplicationSourceShipper Thread for wal group: 9e5740704f6b%2C36187%2C1733440777223 2024-12-05T23:20:44,366 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1.replicationSource.wal-reader.9e5740704f6b%2C36187%2C1733440777223,1 {}] regionserver.WALEntryStream(260): Creating new reader hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/WALs/9e5740704f6b,36187,1733440777223/9e5740704f6b%2C36187%2C1733440777223.1733440780716, startPosition=0, beingWritten=true 2024-12-05T23:20:44,366 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:44,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=252, resume processing ppid=251 2024-12-05T23:20:44,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=252, ppid=251, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 190 msec 2024-12-05T23:20:44,368 INFO [PEWorker-1 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-05T23:20:44,369 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=251, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 202 msec 2024-12-05T23:20:44,373 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1.replicationSource.wal-reader.9e5740704f6b%2C36187%2C1733440777223,1 {}] regionserver.ReplicationSourceWALReader(181): Read 0 WAL entries eligible for replication 2024-12-05T23:20:44,373 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1.replicationSource.wal-reader.9e5740704f6b%2C36187%2C1733440777223,1 {}] regionserver.WALEntryStream(223): Reset reader hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/WALs/9e5740704f6b,36187,1733440777223/9e5740704f6b%2C36187%2C1733440777223.1733440780716 to pos 589, reset compression=false 2024-12-05T23:20:44,373 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1.replicationSource.shipper9e5740704f6b%2C36187%2C1733440777223,1 {}] regionserver.ReplicationSourceShipper(110): Shipper from source 1 got entry batch from reader: WALEntryBatch [walEntries=[], lastWalPath=hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/WALs/9e5740704f6b,36187,1733440777223/9e5740704f6b%2C36187%2C1733440777223.1733440780716, lastWalPosition=589, nbRowKeys=0, nbHFiles=0, heapSize=0, lastSeqIds={}, endOfFile=false,usedBufferSize=0] 2024-12-05T23:20:44,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=251 2024-12-05T23:20:44,486 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:44,487 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=2, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-05T23:20:44,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=253, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-05T23:20:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=253 2024-12-05T23:20:44,488 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] ... 13 more 2024-12-05T23:20:44,489 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=253, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:44,490 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=253, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=2 msec 2024-12-05T23:20:44,490 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:44,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=253 2024-12-05T23:20:44,596 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 2 failed with Invalid cluster key: 2024-12-05T23:20:44,597 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-05T23:20:44,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=254, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:44,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-12-05T23:20:44,599 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:44,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-05T23:20:44,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:44,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-05T23:20:44,602 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=255, ppid=254, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-05T23:20:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-12-05T23:20:44,753 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:44,754 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36187 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=255 2024-12-05T23:20:44,754 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=255}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-05T23:20:44,754 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=255}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-05T23:20:44,754 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=255}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-05T23:20:44,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-12-05T23:20:45,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-12-05T23:20:45,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-12-05T23:20:45,755 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1.replicationSource.shipper9e5740704f6b%2C36187%2C1733440777223,1 {}] regionserver.ReplicationSourceShipper(123): Interrupted while waiting for next replication entry batch java.lang.InterruptedException: null at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1681) ~[?:?] at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) ~[?:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.poll(ReplicationSourceWALReader.java:313) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceShipper.run(ReplicationSourceShipper.java:109) ~[classes/:?] 2024-12-05T23:20:45,755 WARN [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1.replicationSource.wal-reader.9e5740704f6b%2C36187%2C1733440777223,1 {}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:131) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:166) ~[classes/:?] 2024-12-05T23:20:46,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-12-05T23:20:46,755 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=255}] regionserver.ReplicationSource(711): peerId=1, ReplicationSourceWorker RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0.replicationSource,1.replicationSource.shipper9e5740704f6b%2C36187%2C1733440777223,1 terminated 2024-12-05T23:20:46,755 INFO [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=255}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-05T23:20:46,756 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=255}] zookeeper.ZKUtil(111): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:46,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1/9e5740704f6b%2C36187%2C1733440777223.1733440780716 2024-12-05T23:20:46,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:46,758 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/9e5740704f6b,36187,1733440777223/1 2024-12-05T23:20:46,758 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=255}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:46,758 DEBUG [RS_REFRESH_PEER-regionserver/9e5740704f6b:0-0 {event_type=RS_REFRESH_PEER, pid=255}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=255 2024-12-05T23:20:46,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(4106): Remote procedure done, pid=255 2024-12-05T23:20:46,761 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 9e5740704f6b,36187,1733440777223 suceeded 2024-12-05T23:20:46,762 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=255, resume processing ppid=254 2024-12-05T23:20:46,762 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=255, ppid=254, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 2.1590 sec 2024-12-05T23:20:46,765 DEBUG [PEWorker-1 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-05T23:20:46,765 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-05T23:20:46,766 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=254, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 2.1680 sec 2024-12-05T23:20:47,971 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-05T23:20:48,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-12-05T23:20:48,767 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-05T23:20:48,767 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-05T23:20:48,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] procedure2.ProcedureExecutor(1098): Stored pid=256, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-05T23:20:48,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=256 2024-12-05T23:20:48,769 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-05T23:20:48,770 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=256, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-05T23:20:48,771 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=256, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-05T23:20:48,772 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-05T23:20:48,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=38841 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=256 2024-12-05T23:20:48,877 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-05T23:20:48,890 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[1] Thread=217 (was 217), OpenFileDescriptor=445 (was 442) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=556 (was 579), ProcessCount=11 (was 11), AvailableMemoryMB=2628 (was 2794) 2024-12-05T23:20:48,890 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-12-05T23:20:48,890 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.client.TestAsyncAdminBase.tearDownAfterClass(TestAsyncAdminBase.java:96) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-05T23:20:48,891 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:48,891 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1e2eb297 to 127.0.0.1:50903 2024-12-05T23:20:48,891 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-12-05T23:20:48,892 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1ac37e33 to 127.0.0.1:50903 2024-12-05T23:20:48,892 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:48,892 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-05T23:20:48,892 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1646873123, stopped=false 2024-12-05T23:20:48,892 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=9e5740704f6b,38841,1733440775912 2024-12-05T23:20:48,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T23:20:48,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T23:20:48,894 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-12-05T23:20:48,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-05T23:20:48,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T23:20:48,895 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T23:20:48,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:48,895 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-05T23:20:48,895 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '9e5740704f6b,36187,1733440777223' ***** 2024-12-05T23:20:48,895 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-12-05T23:20:48,895 INFO [RS:0;9e5740704f6b:36187 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-05T23:20:48,895 INFO [RS:0;9e5740704f6b:36187 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-05T23:20:48,895 INFO [RS:0;9e5740704f6b:36187 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-05T23:20:48,896 INFO [RS:0;9e5740704f6b:36187 {}] regionserver.HRegionServer(3579): Received CLOSE for 4cc297f79f625a9804bd9a56700799f5 2024-12-05T23:20:48,896 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-12-05T23:20:48,900 INFO [RS:0;9e5740704f6b:36187 {}] regionserver.HRegionServer(1224): stopping server 9e5740704f6b,36187,1733440777223 2024-12-05T23:20:48,900 DEBUG [RS:0;9e5740704f6b:36187 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:48,900 INFO [RS:0;9e5740704f6b:36187 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-05T23:20:48,900 INFO [RS:0;9e5740704f6b:36187 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-05T23:20:48,900 INFO [RS:0;9e5740704f6b:36187 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-05T23:20:48,901 INFO [RS:0;9e5740704f6b:36187 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-12-05T23:20:48,901 INFO [RS:0;9e5740704f6b:36187 {}] regionserver.HRegionServer(1599): Waiting on 2 regions to close 2024-12-05T23:20:48,901 DEBUG [RS:0;9e5740704f6b:36187 {}] regionserver.HRegionServer(1603): Online Regions={1588230740=hbase:meta,,1.1588230740, 4cc297f79f625a9804bd9a56700799f5=hbase:namespace,,1733440781386.4cc297f79f625a9804bd9a56700799f5.} 2024-12-05T23:20:48,901 DEBUG [RS_CLOSE_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-05T23:20:48,901 INFO [RS_CLOSE_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-05T23:20:48,901 DEBUG [RS_CLOSE_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 4cc297f79f625a9804bd9a56700799f5, disabling compactions & flushes 2024-12-05T23:20:48,901 DEBUG [RS:0;9e5740704f6b:36187 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 4cc297f79f625a9804bd9a56700799f5 2024-12-05T23:20:48,901 DEBUG [RS_CLOSE_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-05T23:20:48,901 DEBUG [RS_CLOSE_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-05T23:20:48,901 INFO [RS_CLOSE_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733440781386.4cc297f79f625a9804bd9a56700799f5. 2024-12-05T23:20:48,901 DEBUG [RS_CLOSE_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-05T23:20:48,901 DEBUG [RS_CLOSE_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733440781386.4cc297f79f625a9804bd9a56700799f5. 2024-12-05T23:20:48,901 DEBUG [RS_CLOSE_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733440781386.4cc297f79f625a9804bd9a56700799f5. after waiting 0 ms 2024-12-05T23:20:48,901 DEBUG [RS_CLOSE_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733440781386.4cc297f79f625a9804bd9a56700799f5. 2024-12-05T23:20:48,902 INFO [RS_CLOSE_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 4cc297f79f625a9804bd9a56700799f5 1/1 column families, dataSize=78 B heapSize=488 B 2024-12-05T23:20:48,902 INFO [RS_CLOSE_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=1.23 KB heapSize=2.87 KB 2024-12-05T23:20:48,972 INFO [regionserver/9e5740704f6b:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T23:20:48,979 DEBUG [RS_CLOSE_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/data/hbase/namespace/4cc297f79f625a9804bd9a56700799f5/.tmp/info/6da10e3e7a6846688803fae99cb6879c is 45, key is default/info:d/1733440782859/Put/seqid=0 2024-12-05T23:20:48,979 DEBUG [RS_CLOSE_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/data/hbase/meta/1588230740/.tmp/info/c27fe4a7010442ef95a240d80eb32596 is 143, key is hbase:namespace,,1733440781386.4cc297f79f625a9804bd9a56700799f5./info:regioninfo/1733440782674/Put/seqid=0 2024-12-05T23:20:49,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741838_1014 (size=6595) 2024-12-05T23:20:49,033 INFO [RS_CLOSE_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.14 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/data/hbase/meta/1588230740/.tmp/info/c27fe4a7010442ef95a240d80eb32596 2024-12-05T23:20:49,050 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741837_1013 (size=5037) 2024-12-05T23:20:49,054 INFO [RS_CLOSE_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/data/hbase/namespace/4cc297f79f625a9804bd9a56700799f5/.tmp/info/6da10e3e7a6846688803fae99cb6879c 2024-12-05T23:20:49,102 DEBUG [RS:0;9e5740704f6b:36187 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, 4cc297f79f625a9804bd9a56700799f5 2024-12-05T23:20:49,124 DEBUG [RS_CLOSE_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/data/hbase/namespace/4cc297f79f625a9804bd9a56700799f5/.tmp/info/6da10e3e7a6846688803fae99cb6879c as hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/data/hbase/namespace/4cc297f79f625a9804bd9a56700799f5/info/6da10e3e7a6846688803fae99cb6879c 2024-12-05T23:20:49,146 INFO [RS_CLOSE_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/data/hbase/namespace/4cc297f79f625a9804bd9a56700799f5/info/6da10e3e7a6846688803fae99cb6879c, entries=2, sequenceid=6, filesize=4.9 K 2024-12-05T23:20:49,149 INFO [RS_CLOSE_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 4cc297f79f625a9804bd9a56700799f5 in 247ms, sequenceid=6, compaction requested=false 2024-12-05T23:20:49,157 DEBUG [RS_CLOSE_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/data/hbase/meta/1588230740/.tmp/table/af3e9a0767e9460691a5fa784b47e759 is 51, key is hbase:namespace/table:state/1733440782705/Put/seqid=0 2024-12-05T23:20:49,173 DEBUG [RS_CLOSE_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/data/hbase/namespace/4cc297f79f625a9804bd9a56700799f5/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-05T23:20:49,177 INFO [RS_CLOSE_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1733440781386.4cc297f79f625a9804bd9a56700799f5. 2024-12-05T23:20:49,177 DEBUG [RS_CLOSE_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 4cc297f79f625a9804bd9a56700799f5: 2024-12-05T23:20:49,177 DEBUG [RS_CLOSE_REGION-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1733440781386.4cc297f79f625a9804bd9a56700799f5. 2024-12-05T23:20:49,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741839_1015 (size=5242) 2024-12-05T23:20:49,182 INFO [RS_CLOSE_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=94 B at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/data/hbase/meta/1588230740/.tmp/table/af3e9a0767e9460691a5fa784b47e759 2024-12-05T23:20:49,199 DEBUG [RS_CLOSE_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/data/hbase/meta/1588230740/.tmp/info/c27fe4a7010442ef95a240d80eb32596 as hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/data/hbase/meta/1588230740/info/c27fe4a7010442ef95a240d80eb32596 2024-12-05T23:20:49,212 INFO [RS_CLOSE_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/data/hbase/meta/1588230740/info/c27fe4a7010442ef95a240d80eb32596, entries=10, sequenceid=9, filesize=6.4 K 2024-12-05T23:20:49,214 DEBUG [RS_CLOSE_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/data/hbase/meta/1588230740/.tmp/table/af3e9a0767e9460691a5fa784b47e759 as hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/data/hbase/meta/1588230740/table/af3e9a0767e9460691a5fa784b47e759 2024-12-05T23:20:49,228 INFO [RS_CLOSE_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/data/hbase/meta/1588230740/table/af3e9a0767e9460691a5fa784b47e759, entries=2, sequenceid=9, filesize=5.1 K 2024-12-05T23:20:49,230 INFO [RS_CLOSE_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~1.23 KB/1264, heapSize ~2.59 KB/2648, currentSize=0 B/0 for 1588230740 in 327ms, sequenceid=9, compaction requested=false 2024-12-05T23:20:49,246 DEBUG [RS_CLOSE_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/data/hbase/meta/1588230740/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-05T23:20:49,247 DEBUG [RS_CLOSE_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-05T23:20:49,247 INFO [RS_CLOSE_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-05T23:20:49,247 DEBUG [RS_CLOSE_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-05T23:20:49,247 DEBUG [RS_CLOSE_META-regionserver/9e5740704f6b:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-05T23:20:49,302 INFO [RS:0;9e5740704f6b:36187 {}] regionserver.HRegionServer(1250): stopping server 9e5740704f6b,36187,1733440777223; all regions closed. 2024-12-05T23:20:49,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741834_1010 (size=2484) 2024-12-05T23:20:49,315 DEBUG [RS:0;9e5740704f6b:36187 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/oldWALs 2024-12-05T23:20:49,315 INFO [RS:0;9e5740704f6b:36187 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL 9e5740704f6b%2C36187%2C1733440777223.meta:.meta(num 1733440781118) 2024-12-05T23:20:49,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741833_1009 (size=1414) 2024-12-05T23:20:49,319 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(743): complete file /user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/WALs/9e5740704f6b,36187,1733440777223/9e5740704f6b%2C36187%2C1733440777223.1733440780716 not finished, retry = 0 2024-12-05T23:20:49,424 DEBUG [RS:0;9e5740704f6b:36187 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/oldWALs 2024-12-05T23:20:49,426 INFO [RS:0;9e5740704f6b:36187 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL 9e5740704f6b%2C36187%2C1733440777223:(num 1733440780716) 2024-12-05T23:20:49,426 DEBUG [RS:0;9e5740704f6b:36187 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:49,426 INFO [RS:0;9e5740704f6b:36187 {}] regionserver.LeaseManager(133): Closed leases 2024-12-05T23:20:49,435 INFO [RS:0;9e5740704f6b:36187 {}] hbase.ChoreService(370): Chore service for: regionserver/9e5740704f6b:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2024-12-05T23:20:49,436 INFO [regionserver/9e5740704f6b:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-05T23:20:49,436 INFO [RS:0;9e5740704f6b:36187 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:36187 2024-12-05T23:20:49,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-05T23:20:49,444 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9e5740704f6b,36187,1733440777223 2024-12-05T23:20:49,445 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9e5740704f6b,36187,1733440777223] 2024-12-05T23:20:49,445 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 9e5740704f6b,36187,1733440777223; numProcessing=1 2024-12-05T23:20:49,447 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/9e5740704f6b,36187,1733440777223 already deleted, retry=false 2024-12-05T23:20:49,447 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 9e5740704f6b,36187,1733440777223 expired; onlineServers=0 2024-12-05T23:20:49,447 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server '9e5740704f6b,38841,1733440775912' ***** 2024-12-05T23:20:49,447 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-05T23:20:49,448 DEBUG [M:0;9e5740704f6b:38841 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22109b19, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9e5740704f6b/172.17.0.2:0 2024-12-05T23:20:49,448 INFO [M:0;9e5740704f6b:38841 {}] regionserver.HRegionServer(1224): stopping server 9e5740704f6b,38841,1733440775912 2024-12-05T23:20:49,448 INFO [M:0;9e5740704f6b:38841 {}] regionserver.HRegionServer(1250): stopping server 9e5740704f6b,38841,1733440775912; all regions closed. 2024-12-05T23:20:49,448 DEBUG [M:0;9e5740704f6b:38841 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-05T23:20:49,448 DEBUG [M:0;9e5740704f6b:38841 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-05T23:20:49,448 DEBUG [M:0;9e5740704f6b:38841 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-05T23:20:49,448 DEBUG [master/9e5740704f6b:0:becomeActiveMaster-HFileCleaner.large.0-1733440780294 {}] cleaner.HFileCleaner(306): Exit Thread[master/9e5740704f6b:0:becomeActiveMaster-HFileCleaner.large.0-1733440780294,5,FailOnTimeoutGroup] 2024-12-05T23:20:49,448 DEBUG [master/9e5740704f6b:0:becomeActiveMaster-HFileCleaner.small.0-1733440780301 {}] cleaner.HFileCleaner(306): Exit Thread[master/9e5740704f6b:0:becomeActiveMaster-HFileCleaner.small.0-1733440780301,5,FailOnTimeoutGroup] 2024-12-05T23:20:49,448 INFO [M:0;9e5740704f6b:38841 {}] hbase.ChoreService(370): Chore service for: master/9e5740704f6b:0 had [] on shutdown 2024-12-05T23:20:49,448 DEBUG [M:0;9e5740704f6b:38841 {}] master.HMaster(1733): Stopping service threads 2024-12-05T23:20:49,448 INFO [M:0;9e5740704f6b:38841 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-05T23:20:49,449 INFO [M:0;9e5740704f6b:38841 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-05T23:20:49,449 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-05T23:20:49,450 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-05T23:20:49,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-05T23:20:49,453 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-05T23:20:49,453 DEBUG [M:0;9e5740704f6b:38841 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/master already deleted, retry=false 2024-12-05T23:20:49,453 DEBUG [M:0;9e5740704f6b:38841 {}] master.ActiveMasterManager(353): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Failed delete of our master address node; KeeperErrorCode = NoNode for /hbase/master 2024-12-05T23:20:49,453 INFO [M:0;9e5740704f6b:38841 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-12-05T23:20:49,453 INFO [M:0;9e5740704f6b:38841 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-05T23:20:49,454 DEBUG [M:0;9e5740704f6b:38841 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-05T23:20:49,454 INFO [M:0;9e5740704f6b:38841 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T23:20:49,454 DEBUG [M:0;9e5740704f6b:38841 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T23:20:49,454 DEBUG [M:0;9e5740704f6b:38841 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-05T23:20:49,454 DEBUG [M:0;9e5740704f6b:38841 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T23:20:49,454 INFO [M:0;9e5740704f6b:38841 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=746.31 KB heapSize=895.09 KB 2024-12-05T23:20:49,454 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-05T23:20:49,479 DEBUG [M:0;9e5740704f6b:38841 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cb3c2dc03e374f98b4901cee34ba3350 is 82, key is hbase:meta,,1/info:regioninfo/1733440781274/Put/seqid=0 2024-12-05T23:20:49,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741840_1016 (size=5672) 2024-12-05T23:20:49,491 INFO [M:0;9e5740704f6b:38841 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=1809 (bloomFilter=true), to=hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cb3c2dc03e374f98b4901cee34ba3350 2024-12-05T23:20:49,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:49,547 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:36187-0x10184ed0ca40001, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:49,547 INFO [RS:0;9e5740704f6b:36187 {}] regionserver.HRegionServer(1307): Exiting; stopping=9e5740704f6b,36187,1733440777223; zookeeper connection closed. 2024-12-05T23:20:49,548 DEBUG [M:0;9e5740704f6b:38841 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fa04d2db0a3e4f7b813cb802c37fc90d is 2089, key is \x00\x00\x00\x00\x00\x00\x00\x9D/proc:d/1733440823175/Put/seqid=0 2024-12-05T23:20:49,551 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1f39646f {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1f39646f 2024-12-05T23:20:49,551 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-05T23:20:49,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741841_1017 (size=167134) 2024-12-05T23:20:49,953 INFO [M:0;9e5740704f6b:38841 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=743.75 KB at sequenceid=1809 (bloomFilter=true), to=hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fa04d2db0a3e4f7b813cb802c37fc90d 2024-12-05T23:20:49,964 INFO [M:0;9e5740704f6b:38841 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fa04d2db0a3e4f7b813cb802c37fc90d 2024-12-05T23:20:49,982 DEBUG [M:0;9e5740704f6b:38841 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/589198f38887492eba51da91cc6b3013 is 69, key is 9e5740704f6b,36187,1733440777223/rs:state/1733440780403/Put/seqid=0 2024-12-05T23:20:49,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741842_1018 (size=5156) 2024-12-05T23:20:50,388 INFO [M:0;9e5740704f6b:38841 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=1809 (bloomFilter=true), to=hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/589198f38887492eba51da91cc6b3013 2024-12-05T23:20:50,427 DEBUG [M:0;9e5740704f6b:38841 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/092084c689124456a46df90687c32621 is 68, key is replication_peer_modification_on/state:d/1733440848877/Put/seqid=0 2024-12-05T23:20:50,435 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741843_1019 (size=5154) 2024-12-05T23:20:50,837 INFO [M:0;9e5740704f6b:38841 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2 KB at sequenceid=1809 (bloomFilter=true), to=hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/092084c689124456a46df90687c32621 2024-12-05T23:20:50,850 DEBUG [M:0;9e5740704f6b:38841 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cb3c2dc03e374f98b4901cee34ba3350 as hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cb3c2dc03e374f98b4901cee34ba3350 2024-12-05T23:20:50,858 INFO [M:0;9e5740704f6b:38841 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cb3c2dc03e374f98b4901cee34ba3350, entries=8, sequenceid=1809, filesize=5.5 K 2024-12-05T23:20:50,860 DEBUG [M:0;9e5740704f6b:38841 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/fa04d2db0a3e4f7b813cb802c37fc90d as hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fa04d2db0a3e4f7b813cb802c37fc90d 2024-12-05T23:20:50,867 INFO [M:0;9e5740704f6b:38841 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for fa04d2db0a3e4f7b813cb802c37fc90d 2024-12-05T23:20:50,868 INFO [M:0;9e5740704f6b:38841 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/fa04d2db0a3e4f7b813cb802c37fc90d, entries=256, sequenceid=1809, filesize=163.2 K 2024-12-05T23:20:50,869 DEBUG [M:0;9e5740704f6b:38841 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/589198f38887492eba51da91cc6b3013 as hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/589198f38887492eba51da91cc6b3013 2024-12-05T23:20:50,878 INFO [M:0;9e5740704f6b:38841 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/589198f38887492eba51da91cc6b3013, entries=1, sequenceid=1809, filesize=5.0 K 2024-12-05T23:20:50,879 DEBUG [M:0;9e5740704f6b:38841 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/092084c689124456a46df90687c32621 as hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/092084c689124456a46df90687c32621 2024-12-05T23:20:50,888 INFO [M:0;9e5740704f6b:38841 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:44761/user/jenkins/test-data/844309b7-a463-3a6d-d1b3-9284c26a59a9/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/092084c689124456a46df90687c32621, entries=1, sequenceid=1809, filesize=5.0 K 2024-12-05T23:20:50,889 INFO [M:0;9e5740704f6b:38841 {}] regionserver.HRegion(3040): Finished flush of dataSize ~746.31 KB/764219, heapSize ~895.03 KB/916512, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1434ms, sequenceid=1809, compaction requested=false 2024-12-05T23:20:50,890 INFO [M:0;9e5740704f6b:38841 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-05T23:20:50,890 DEBUG [M:0;9e5740704f6b:38841 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-05T23:20:50,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:38377 is added to blk_1073741830_1006 (size=883029) 2024-12-05T23:20:50,897 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-05T23:20:50,897 INFO [M:0;9e5740704f6b:38841 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-12-05T23:20:50,897 INFO [M:0;9e5740704f6b:38841 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:38841 2024-12-05T23:20:50,902 DEBUG [M:0;9e5740704f6b:38841 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/9e5740704f6b,38841,1733440775912 already deleted, retry=false 2024-12-05T23:20:51,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:51,006 INFO [M:0;9e5740704f6b:38841 {}] regionserver.HRegionServer(1307): Exiting; stopping=9e5740704f6b,38841,1733440775912; zookeeper connection closed. 2024-12-05T23:20:51,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:38841-0x10184ed0ca40000, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:51,109 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10184ed0ca40019, quorum=127.0.0.1:50903, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-05T23:20:51,114 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@6e50d5cf{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-05T23:20:51,117 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@291745d8{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T23:20:51,117 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T23:20:51,117 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@48604e1{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T23:20:51,117 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@a00c17f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/hadoop.log.dir/,STOPPED} 2024-12-05T23:20:51,119 WARN [BP-81780034-172.17.0.2-1733440770420 heartbeating to localhost/127.0.0.1:44761 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-05T23:20:51,119 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-05T23:20:51,120 WARN [BP-81780034-172.17.0.2-1733440770420 heartbeating to localhost/127.0.0.1:44761 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-81780034-172.17.0.2-1733440770420 (Datanode Uuid d04413b2-2858-4c3a-905b-f46e3d9f905c) service to localhost/127.0.0.1:44761 2024-12-05T23:20:51,120 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-05T23:20:51,121 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/cluster_ea30cb3b-51ca-5a5b-9a30-c0d33a20f2ca/dfs/data/data1/current/BP-81780034-172.17.0.2-1733440770420 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T23:20:51,121 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/cluster_ea30cb3b-51ca-5a5b-9a30-c0d33a20f2ca/dfs/data/data2/current/BP-81780034-172.17.0.2-1733440770420 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-05T23:20:51,122 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-05T23:20:51,130 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2b5eba6b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-05T23:20:51,130 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@79c1d5da{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-05T23:20:51,130 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-05T23:20:51,131 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@f2948e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-05T23:20:51,131 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1d9ceadd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/dc114973-2997-d603-a899-2027398e60da/hadoop.log.dir/,STOPPED} 2024-12-05T23:20:51,144 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-12-05T23:20:51,164 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down