2024-12-02 14:20:33,391 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7ae0a9ec 2024-12-02 14:20:33,404 main DEBUG Took 0.010625 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-02 14:20:33,404 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-02 14:20:33,405 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-02 14:20:33,406 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-02 14:20:33,407 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 14:20:33,421 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-02 14:20:33,433 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:20:33,434 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 14:20:33,435 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:20:33,435 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 14:20:33,436 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:20:33,436 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 14:20:33,437 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:20:33,437 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 14:20:33,438 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:20:33,438 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 14:20:33,439 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:20:33,439 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 14:20:33,439 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:20:33,440 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 14:20:33,440 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:20:33,440 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 14:20:33,441 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:20:33,441 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 14:20:33,442 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:20:33,442 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 14:20:33,442 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:20:33,442 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 14:20:33,443 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:20:33,443 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-02 14:20:33,443 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:20:33,444 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-02 14:20:33,445 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-02 14:20:33,446 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-02 14:20:33,448 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-02 14:20:33,448 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-02 14:20:33,449 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-02 14:20:33,450 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-02 14:20:33,458 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-02 14:20:33,461 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-02 14:20:33,462 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-02 14:20:33,463 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-02 14:20:33,463 main DEBUG createAppenders(={Console}) 2024-12-02 14:20:33,464 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7ae0a9ec initialized 2024-12-02 14:20:33,464 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7ae0a9ec 2024-12-02 14:20:33,465 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7ae0a9ec OK. 2024-12-02 14:20:33,465 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-02 14:20:33,465 main DEBUG OutputStream closed 2024-12-02 14:20:33,466 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-02 14:20:33,466 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-02 14:20:33,466 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4efc180e OK 2024-12-02 14:20:33,543 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-02 14:20:33,547 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-02 14:20:33,548 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-02 14:20:33,550 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-02 14:20:33,551 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-02 14:20:33,552 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-02 14:20:33,552 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-02 14:20:33,553 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-02 14:20:33,554 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-02 14:20:33,554 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-02 14:20:33,555 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-02 14:20:33,555 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-02 14:20:33,556 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-02 14:20:33,556 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-02 14:20:33,557 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-02 14:20:33,557 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-02 14:20:33,557 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-02 14:20:33,559 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-02 14:20:33,562 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-02 14:20:33,563 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@39de3d36) with optional ClassLoader: null 2024-12-02 14:20:33,564 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-02 14:20:33,565 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@39de3d36] started OK. 2024-12-02T14:20:33,908 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226 2024-12-02 14:20:33,914 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-02 14:20:33,914 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-02T14:20:33,946 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestAsyncReplicationAdminApi timeout: 26 mins 2024-12-02T14:20:33,970 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-02T14:20:33,993 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/cluster_c3337032-8624-86a2-153d-f1cb3af65c88, deleteOnExit=true 2024-12-02T14:20:33,994 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-12-02T14:20:33,997 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/test.cache.data in system properties and HBase conf 2024-12-02T14:20:33,998 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/hadoop.tmp.dir in system properties and HBase conf 2024-12-02T14:20:33,999 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/hadoop.log.dir in system properties and HBase conf 2024-12-02T14:20:34,000 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-02T14:20:34,001 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-02T14:20:34,001 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-12-02T14:20:34,105 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-02T14:20:34,216 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-02T14:20:34,220 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-02T14:20:34,220 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-02T14:20:34,221 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-02T14:20:34,221 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T14:20:34,222 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-02T14:20:34,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-02T14:20:34,223 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-02T14:20:34,225 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T14:20:34,226 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-02T14:20:34,227 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/nfs.dump.dir in system properties and HBase conf 2024-12-02T14:20:34,227 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/java.io.tmpdir in system properties and HBase conf 2024-12-02T14:20:34,228 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-02T14:20:34,228 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-02T14:20:34,229 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-02T14:20:35,177 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-02T14:20:35,304 INFO [Time-limited test {}] log.Log(170): Logging initialized @2572ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-02T14:20:35,414 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T14:20:35,487 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T14:20:35,517 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T14:20:35,518 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T14:20:35,520 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T14:20:35,535 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T14:20:35,539 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@33fb8453{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/hadoop.log.dir/,AVAILABLE} 2024-12-02T14:20:35,541 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7b133317{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T14:20:35,781 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1441d181{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/java.io.tmpdir/jetty-localhost-34999-hadoop-hdfs-3_4_1-tests_jar-_-any-13578954063951342928/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T14:20:35,788 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@135deabe{HTTP/1.1, (http/1.1)}{localhost:34999} 2024-12-02T14:20:35,789 INFO [Time-limited test {}] server.Server(415): Started @3058ms 2024-12-02T14:20:36,331 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-02T14:20:36,345 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-02T14:20:36,354 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-02T14:20:36,355 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-02T14:20:36,355 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-02T14:20:36,356 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@39bd1973{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/hadoop.log.dir/,AVAILABLE} 2024-12-02T14:20:36,357 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@43c2f64f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-02T14:20:36,474 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@30338fac{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/java.io.tmpdir/jetty-localhost-34485-hadoop-hdfs-3_4_1-tests_jar-_-any-1694979172818551312/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T14:20:36,475 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@7f8511bb{HTTP/1.1, (http/1.1)}{localhost:34485} 2024-12-02T14:20:36,476 INFO [Time-limited test {}] server.Server(415): Started @3744ms 2024-12-02T14:20:36,535 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-02T14:20:37,113 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/cluster_c3337032-8624-86a2-153d-f1cb3af65c88/data/data2/current/BP-1636995057-172.17.0.2-1733149234894/current, will proceed with Du for space computation calculation, 2024-12-02T14:20:37,114 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/cluster_c3337032-8624-86a2-153d-f1cb3af65c88/data/data1/current/BP-1636995057-172.17.0.2-1733149234894/current, will proceed with Du for space computation calculation, 2024-12-02T14:20:37,236 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-02T14:20:37,306 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1972cb6d0812a9b0 with lease ID 0x9240be6132fa164e: Processing first storage report for DS-a700cf0b-ff60-4712-a70e-6a39b5d11e1b from datanode DatanodeRegistration(127.0.0.1:37975, datanodeUuid=168ed01c-ffa7-4f2a-95c0-899bedb822af, infoPort=36835, infoSecurePort=0, ipcPort=41495, storageInfo=lv=-57;cid=testClusterID;nsid=1444457515;c=1733149234894) 2024-12-02T14:20:37,307 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1972cb6d0812a9b0 with lease ID 0x9240be6132fa164e: from storage DS-a700cf0b-ff60-4712-a70e-6a39b5d11e1b node DatanodeRegistration(127.0.0.1:37975, datanodeUuid=168ed01c-ffa7-4f2a-95c0-899bedb822af, infoPort=36835, infoSecurePort=0, ipcPort=41495, storageInfo=lv=-57;cid=testClusterID;nsid=1444457515;c=1733149234894), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-02T14:20:37,307 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x1972cb6d0812a9b0 with lease ID 0x9240be6132fa164e: Processing first storage report for DS-8dedf7a2-9a55-4505-a212-0ce7733b2c0c from datanode DatanodeRegistration(127.0.0.1:37975, datanodeUuid=168ed01c-ffa7-4f2a-95c0-899bedb822af, infoPort=36835, infoSecurePort=0, ipcPort=41495, storageInfo=lv=-57;cid=testClusterID;nsid=1444457515;c=1733149234894) 2024-12-02T14:20:37,308 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x1972cb6d0812a9b0 with lease ID 0x9240be6132fa164e: from storage DS-8dedf7a2-9a55-4505-a212-0ce7733b2c0c node DatanodeRegistration(127.0.0.1:37975, datanodeUuid=168ed01c-ffa7-4f2a-95c0-899bedb822af, infoPort=36835, infoSecurePort=0, ipcPort=41495, storageInfo=lv=-57;cid=testClusterID;nsid=1444457515;c=1733149234894), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-12-02T14:20:37,330 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226 2024-12-02T14:20:37,422 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/cluster_c3337032-8624-86a2-153d-f1cb3af65c88/zookeeper_0, clientPort=53873, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/cluster_c3337032-8624-86a2-153d-f1cb3af65c88/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/cluster_c3337032-8624-86a2-153d-f1cb3af65c88/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-02T14:20:37,433 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53873 2024-12-02T14:20:37,447 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T14:20:37,451 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T14:20:37,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37975 is added to blk_1073741825_1001 (size=7) 2024-12-02T14:20:38,151 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4 with version=8 2024-12-02T14:20:38,151 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/hbase-staging 2024-12-02T14:20:38,273 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-02T14:20:38,501 INFO [Time-limited test {}] client.ConnectionUtils(128): master/a3a61c9ba14f:0 server-side Connection retries=6 2024-12-02T14:20:38,511 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T14:20:38,512 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T14:20:38,517 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T14:20:38,517 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T14:20:38,518 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T14:20:38,684 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-12-02T14:20:38,749 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-02T14:20:38,762 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-02T14:20:38,767 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T14:20:38,803 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 105009 (auto-detected) 2024-12-02T14:20:38,804 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-02T14:20:38,833 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:36883 2024-12-02T14:20:38,864 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:36883 connecting to ZooKeeper ensemble=127.0.0.1:53873 2024-12-02T14:20:38,905 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:368830x0, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T14:20:38,919 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36883-0x1009b4dfd050000 connected 2024-12-02T14:20:38,970 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T14:20:38,973 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T14:20:38,984 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T14:20:38,988 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4, hbase.cluster.distributed=false 2024-12-02T14:20:39,014 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T14:20:39,022 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36883 2024-12-02T14:20:39,025 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36883 2024-12-02T14:20:39,028 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36883 2024-12-02T14:20:39,033 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36883 2024-12-02T14:20:39,036 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36883 2024-12-02T14:20:39,152 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/a3a61c9ba14f:0 server-side Connection retries=6 2024-12-02T14:20:39,154 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T14:20:39,154 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-02T14:20:39,154 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-02T14:20:39,155 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-02T14:20:39,155 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-02T14:20:39,158 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-02T14:20:39,159 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-02T14:20:39,160 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:46013 2024-12-02T14:20:39,163 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:46013 connecting to ZooKeeper ensemble=127.0.0.1:53873 2024-12-02T14:20:39,165 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T14:20:39,170 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T14:20:39,181 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:460130x0, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-02T14:20:39,183 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:460130x0, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T14:20:39,188 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-02T14:20:39,190 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:46013-0x1009b4dfd050001 connected 2024-12-02T14:20:39,200 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-02T14:20:39,205 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46013-0x1009b4dfd050001, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-02T14:20:39,213 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:46013-0x1009b4dfd050001, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-02T14:20:39,217 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46013 2024-12-02T14:20:39,218 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46013 2024-12-02T14:20:39,218 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46013 2024-12-02T14:20:39,219 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46013 2024-12-02T14:20:39,220 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46013 2024-12-02T14:20:39,249 DEBUG [M:0;a3a61c9ba14f:36883 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;a3a61c9ba14f:36883 2024-12-02T14:20:39,250 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/a3a61c9ba14f,36883,1733149238328 2024-12-02T14:20:39,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46013-0x1009b4dfd050001, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T14:20:39,256 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T14:20:39,261 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/a3a61c9ba14f,36883,1733149238328 2024-12-02T14:20:39,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:20:39,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46013-0x1009b4dfd050001, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-02T14:20:39,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46013-0x1009b4dfd050001, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:20:39,284 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-02T14:20:39,285 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/a3a61c9ba14f,36883,1733149238328 from backup master directory 2024-12-02T14:20:39,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/a3a61c9ba14f,36883,1733149238328 2024-12-02T14:20:39,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46013-0x1009b4dfd050001, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T14:20:39,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-02T14:20:39,291 WARN [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T14:20:39,292 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=a3a61c9ba14f,36883,1733149238328 2024-12-02T14:20:39,294 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-02T14:20:39,296 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-02T14:20:39,355 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/hbase.id] with ID: 5762d0f7-da11-4ede-9d35-feade635cfff 2024-12-02T14:20:39,355 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/.tmp/hbase.id 2024-12-02T14:20:39,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37975 is added to blk_1073741826_1002 (size=42) 2024-12-02T14:20:39,368 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/.tmp/hbase.id]:[hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/hbase.id] 2024-12-02T14:20:39,424 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-02T14:20:39,431 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-12-02T14:20:39,458 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 23ms. 2024-12-02T14:20:39,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46013-0x1009b4dfd050001, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:20:39,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:20:39,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37975 is added to blk_1073741827_1003 (size=196) 2024-12-02T14:20:39,509 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T14:20:39,512 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-02T14:20:39,532 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:39,538 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T14:20:39,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37975 is added to blk_1073741828_1004 (size=1189) 2024-12-02T14:20:39,991 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/data/master/store 2024-12-02T14:20:40,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37975 is added to blk_1073741829_1005 (size=34) 2024-12-02T14:20:40,422 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-02T14:20:40,426 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T14:20:40,427 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T14:20:40,428 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T14:20:40,428 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T14:20:40,430 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T14:20:40,430 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T14:20:40,430 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T14:20:40,432 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733149240427Disabling compacts and flushes for region at 1733149240427Disabling writes for close at 1733149240430 (+3 ms)Writing region close event to WAL at 1733149240430Closed at 1733149240430 2024-12-02T14:20:40,434 WARN [master/a3a61c9ba14f:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/data/master/store/.initializing 2024-12-02T14:20:40,434 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/WALs/a3a61c9ba14f,36883,1733149238328 2024-12-02T14:20:40,443 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T14:20:40,460 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a3a61c9ba14f%2C36883%2C1733149238328, suffix=, logDir=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/WALs/a3a61c9ba14f,36883,1733149238328, archiveDir=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/oldWALs, maxLogs=10 2024-12-02T14:20:40,491 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/WALs/a3a61c9ba14f,36883,1733149238328/a3a61c9ba14f%2C36883%2C1733149238328.1733149240466, exclude list is [], retry=0 2024-12-02T14:20:40,512 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37975,DS-a700cf0b-ff60-4712-a70e-6a39b5d11e1b,DISK] 2024-12-02T14:20:40,515 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-02T14:20:40,554 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/WALs/a3a61c9ba14f,36883,1733149238328/a3a61c9ba14f%2C36883%2C1733149238328.1733149240466 2024-12-02T14:20:40,555 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36835:36835)] 2024-12-02T14:20:40,556 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-02T14:20:40,556 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T14:20:40,560 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T14:20:40,562 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T14:20:40,608 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T14:20:40,633 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-02T14:20:40,637 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:40,639 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T14:20:40,640 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T14:20:40,645 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-02T14:20:40,645 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:40,646 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:20:40,647 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T14:20:40,650 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-02T14:20:40,651 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:40,652 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:20:40,652 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-02T14:20:40,655 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-02T14:20:40,656 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:40,657 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:20:40,658 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T14:20:40,662 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T14:20:40,664 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-02T14:20:40,671 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T14:20:40,672 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T14:20:40,677 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T14:20:40,681 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-02T14:20:40,686 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T14:20:40,687 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67089464, jitterRate=-2.8908252716064453E-4}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T14:20:40,693 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1733149240575Initializing all the Stores at 1733149240578 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733149240579 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149240579Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149240580 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149240580Cleaning up temporary data from old regions at 1733149240672 (+92 ms)Region opened successfully at 1733149240693 (+21 ms) 2024-12-02T14:20:40,694 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-02T14:20:40,733 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c53bfaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a3a61c9ba14f/172.17.0.2:0 2024-12-02T14:20:40,771 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-12-02T14:20:40,781 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-02T14:20:40,781 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-02T14:20:40,784 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-02T14:20:40,785 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-12-02T14:20:40,791 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-12-02T14:20:40,791 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-02T14:20:40,821 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-02T14:20:40,830 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-02T14:20:40,832 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-12-02T14:20:40,834 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-02T14:20:40,836 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-02T14:20:40,838 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-12-02T14:20:40,841 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-02T14:20:40,844 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-02T14:20:40,846 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-12-02T14:20:40,847 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-02T14:20:40,848 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-12-02T14:20:40,870 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-02T14:20:40,879 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-02T14:20:40,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46013-0x1009b4dfd050001, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T14:20:40,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-02T14:20:40,888 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46013-0x1009b4dfd050001, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:20:40,890 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:20:40,894 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=a3a61c9ba14f,36883,1733149238328, sessionid=0x1009b4dfd050000, setting cluster-up flag (Was=false) 2024-12-02T14:20:40,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:20:40,909 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46013-0x1009b4dfd050001, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:20:40,915 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-02T14:20:40,916 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a3a61c9ba14f,36883,1733149238328 2024-12-02T14:20:40,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:20:40,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46013-0x1009b4dfd050001, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:20:40,928 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-02T14:20:40,930 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=a3a61c9ba14f,36883,1733149238328 2024-12-02T14:20:40,937 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-12-02T14:20:41,015 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-12-02T14:20:41,026 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-12-02T14:20:41,033 INFO [RS:0;a3a61c9ba14f:46013 {}] regionserver.HRegionServer(746): ClusterId : 5762d0f7-da11-4ede-9d35-feade635cfff 2024-12-02T14:20:41,034 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-02T14:20:41,036 DEBUG [RS:0;a3a61c9ba14f:46013 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-02T14:20:41,040 DEBUG [RS:0;a3a61c9ba14f:46013 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-02T14:20:41,040 DEBUG [RS:0;a3a61c9ba14f:46013 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-02T14:20:41,043 DEBUG [RS:0;a3a61c9ba14f:46013 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-02T14:20:41,040 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: a3a61c9ba14f,36883,1733149238328 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-02T14:20:41,044 DEBUG [RS:0;a3a61c9ba14f:46013 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65f02407, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=a3a61c9ba14f/172.17.0.2:0 2024-12-02T14:20:41,048 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/a3a61c9ba14f:0, corePoolSize=5, maxPoolSize=5 2024-12-02T14:20:41,049 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/a3a61c9ba14f:0, corePoolSize=5, maxPoolSize=5 2024-12-02T14:20:41,049 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/a3a61c9ba14f:0, corePoolSize=5, maxPoolSize=5 2024-12-02T14:20:41,049 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/a3a61c9ba14f:0, corePoolSize=5, maxPoolSize=5 2024-12-02T14:20:41,049 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/a3a61c9ba14f:0, corePoolSize=10, maxPoolSize=10 2024-12-02T14:20:41,049 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/a3a61c9ba14f:0, corePoolSize=1, maxPoolSize=1 2024-12-02T14:20:41,050 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/a3a61c9ba14f:0, corePoolSize=2, maxPoolSize=2 2024-12-02T14:20:41,050 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/a3a61c9ba14f:0, corePoolSize=1, maxPoolSize=1 2024-12-02T14:20:41,058 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T14:20:41,059 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-12-02T14:20:41,065 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733149271065 2024-12-02T14:20:41,066 DEBUG [RS:0;a3a61c9ba14f:46013 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;a3a61c9ba14f:46013 2024-12-02T14:20:41,067 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-02T14:20:41,069 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-02T14:20:41,070 INFO [RS:0;a3a61c9ba14f:46013 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-12-02T14:20:41,070 INFO [RS:0;a3a61c9ba14f:46013 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-12-02T14:20:41,070 DEBUG [RS:0;a3a61c9ba14f:46013 {}] regionserver.HRegionServer(832): About to register with Master. 2024-12-02T14:20:41,073 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-02T14:20:41,074 INFO [RS:0;a3a61c9ba14f:46013 {}] regionserver.HRegionServer(2659): reportForDuty to master=a3a61c9ba14f,36883,1733149238328 with port=46013, startcode=1733149239111 2024-12-02T14:20:41,075 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,076 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-02T14:20:41,076 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-02T14:20:41,075 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-02T14:20:41,076 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-02T14:20:41,077 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T14:20:41,089 DEBUG [RS:0;a3a61c9ba14f:46013 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-02T14:20:41,093 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-02T14:20:41,094 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-02T14:20:41,095 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-02T14:20:41,097 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-02T14:20:41,098 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-02T14:20:41,101 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/a3a61c9ba14f:0:becomeActiveMaster-HFileCleaner.large.0-1733149241099,5,FailOnTimeoutGroup] 2024-12-02T14:20:41,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37975 is added to blk_1073741831_1007 (size=1321) 2024-12-02T14:20:41,103 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/a3a61c9ba14f:0:becomeActiveMaster-HFileCleaner.small.0-1733149241102,5,FailOnTimeoutGroup] 2024-12-02T14:20:41,103 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-02T14:20:41,104 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-02T14:20:41,105 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-02T14:20:41,105 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-02T14:20:41,137 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:57903, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-02T14:20:41,143 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36883 {}] master.ServerManager(363): Checking decommissioned status of RegionServer a3a61c9ba14f,46013,1733149239111 2024-12-02T14:20:41,145 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36883 {}] master.ServerManager(517): Registering regionserver=a3a61c9ba14f,46013,1733149239111 2024-12-02T14:20:41,161 DEBUG [RS:0;a3a61c9ba14f:46013 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4 2024-12-02T14:20:41,161 DEBUG [RS:0;a3a61c9ba14f:46013 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:46667 2024-12-02T14:20:41,162 DEBUG [RS:0;a3a61c9ba14f:46013 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-12-02T14:20:41,166 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T14:20:41,167 DEBUG [RS:0;a3a61c9ba14f:46013 {}] zookeeper.ZKUtil(111): regionserver:46013-0x1009b4dfd050001, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/a3a61c9ba14f,46013,1733149239111 2024-12-02T14:20:41,167 WARN [RS:0;a3a61c9ba14f:46013 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-02T14:20:41,167 INFO [RS:0;a3a61c9ba14f:46013 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T14:20:41,168 DEBUG [RS:0;a3a61c9ba14f:46013 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/WALs/a3a61c9ba14f,46013,1733149239111 2024-12-02T14:20:41,169 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [a3a61c9ba14f,46013,1733149239111] 2024-12-02T14:20:41,206 INFO [RS:0;a3a61c9ba14f:46013 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-02T14:20:41,224 INFO [RS:0;a3a61c9ba14f:46013 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-02T14:20:41,230 INFO [RS:0;a3a61c9ba14f:46013 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-02T14:20:41,230 INFO [RS:0;a3a61c9ba14f:46013 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T14:20:41,231 INFO [RS:0;a3a61c9ba14f:46013 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-12-02T14:20:41,238 INFO [RS:0;a3a61c9ba14f:46013 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-12-02T14:20:41,240 INFO [RS:0;a3a61c9ba14f:46013 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-02T14:20:41,240 DEBUG [RS:0;a3a61c9ba14f:46013 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/a3a61c9ba14f:0, corePoolSize=1, maxPoolSize=1 2024-12-02T14:20:41,240 DEBUG [RS:0;a3a61c9ba14f:46013 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/a3a61c9ba14f:0, corePoolSize=1, maxPoolSize=1 2024-12-02T14:20:41,241 DEBUG [RS:0;a3a61c9ba14f:46013 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0, corePoolSize=1, maxPoolSize=1 2024-12-02T14:20:41,241 DEBUG [RS:0;a3a61c9ba14f:46013 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0, corePoolSize=1, maxPoolSize=1 2024-12-02T14:20:41,241 DEBUG [RS:0;a3a61c9ba14f:46013 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/a3a61c9ba14f:0, corePoolSize=1, maxPoolSize=1 2024-12-02T14:20:41,241 DEBUG [RS:0;a3a61c9ba14f:46013 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/a3a61c9ba14f:0, corePoolSize=2, maxPoolSize=2 2024-12-02T14:20:41,241 DEBUG [RS:0;a3a61c9ba14f:46013 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/a3a61c9ba14f:0, corePoolSize=1, maxPoolSize=1 2024-12-02T14:20:41,242 DEBUG [RS:0;a3a61c9ba14f:46013 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/a3a61c9ba14f:0, corePoolSize=1, maxPoolSize=1 2024-12-02T14:20:41,242 DEBUG [RS:0;a3a61c9ba14f:46013 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0, corePoolSize=1, maxPoolSize=1 2024-12-02T14:20:41,242 DEBUG [RS:0;a3a61c9ba14f:46013 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/a3a61c9ba14f:0, corePoolSize=1, maxPoolSize=1 2024-12-02T14:20:41,242 DEBUG [RS:0;a3a61c9ba14f:46013 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/a3a61c9ba14f:0, corePoolSize=1, maxPoolSize=1 2024-12-02T14:20:41,242 DEBUG [RS:0;a3a61c9ba14f:46013 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/a3a61c9ba14f:0, corePoolSize=1, maxPoolSize=1 2024-12-02T14:20:41,243 DEBUG [RS:0;a3a61c9ba14f:46013 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/a3a61c9ba14f:0, corePoolSize=3, maxPoolSize=3 2024-12-02T14:20:41,243 DEBUG [RS:0;a3a61c9ba14f:46013 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/a3a61c9ba14f:0, corePoolSize=3, maxPoolSize=3 2024-12-02T14:20:41,244 INFO [RS:0;a3a61c9ba14f:46013 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T14:20:41,244 INFO [RS:0;a3a61c9ba14f:46013 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-02T14:20:41,244 INFO [RS:0;a3a61c9ba14f:46013 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T14:20:41,244 INFO [RS:0;a3a61c9ba14f:46013 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-02T14:20:41,245 INFO [RS:0;a3a61c9ba14f:46013 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-02T14:20:41,245 INFO [RS:0;a3a61c9ba14f:46013 {}] hbase.ChoreService(168): Chore ScheduledChore name=a3a61c9ba14f,46013,1733149239111-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T14:20:41,274 INFO [RS:0;a3a61c9ba14f:46013 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-02T14:20:41,277 INFO [RS:0;a3a61c9ba14f:46013 {}] hbase.ChoreService(168): Chore ScheduledChore name=a3a61c9ba14f,46013,1733149239111-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T14:20:41,277 INFO [RS:0;a3a61c9ba14f:46013 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T14:20:41,278 INFO [RS:0;a3a61c9ba14f:46013 {}] regionserver.Replication(171): a3a61c9ba14f,46013,1733149239111 started 2024-12-02T14:20:41,304 INFO [RS:0;a3a61c9ba14f:46013 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T14:20:41,304 INFO [RS:0;a3a61c9ba14f:46013 {}] regionserver.HRegionServer(1482): Serving as a3a61c9ba14f,46013,1733149239111, RpcServer on a3a61c9ba14f/172.17.0.2:46013, sessionid=0x1009b4dfd050001 2024-12-02T14:20:41,305 DEBUG [RS:0;a3a61c9ba14f:46013 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-02T14:20:41,305 DEBUG [RS:0;a3a61c9ba14f:46013 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager a3a61c9ba14f,46013,1733149239111 2024-12-02T14:20:41,306 DEBUG [RS:0;a3a61c9ba14f:46013 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a3a61c9ba14f,46013,1733149239111' 2024-12-02T14:20:41,306 DEBUG [RS:0;a3a61c9ba14f:46013 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-02T14:20:41,307 DEBUG [RS:0;a3a61c9ba14f:46013 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-02T14:20:41,309 DEBUG [RS:0;a3a61c9ba14f:46013 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-02T14:20:41,309 DEBUG [RS:0;a3a61c9ba14f:46013 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-02T14:20:41,310 DEBUG [RS:0;a3a61c9ba14f:46013 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager a3a61c9ba14f,46013,1733149239111 2024-12-02T14:20:41,310 DEBUG [RS:0;a3a61c9ba14f:46013 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'a3a61c9ba14f,46013,1733149239111' 2024-12-02T14:20:41,310 DEBUG [RS:0;a3a61c9ba14f:46013 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-02T14:20:41,311 DEBUG [RS:0;a3a61c9ba14f:46013 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-02T14:20:41,312 DEBUG [RS:0;a3a61c9ba14f:46013 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-02T14:20:41,312 INFO [RS:0;a3a61c9ba14f:46013 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-02T14:20:41,312 INFO [RS:0;a3a61c9ba14f:46013 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-02T14:20:41,421 INFO [RS:0;a3a61c9ba14f:46013 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-02T14:20:41,426 INFO [RS:0;a3a61c9ba14f:46013 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a3a61c9ba14f%2C46013%2C1733149239111, suffix=, logDir=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/WALs/a3a61c9ba14f,46013,1733149239111, archiveDir=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/oldWALs, maxLogs=32 2024-12-02T14:20:41,446 DEBUG [RS:0;a3a61c9ba14f:46013 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/WALs/a3a61c9ba14f,46013,1733149239111/a3a61c9ba14f%2C46013%2C1733149239111.1733149241428, exclude list is [], retry=0 2024-12-02T14:20:41,453 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37975,DS-a700cf0b-ff60-4712-a70e-6a39b5d11e1b,DISK] 2024-12-02T14:20:41,463 INFO [RS:0;a3a61c9ba14f:46013 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/WALs/a3a61c9ba14f,46013,1733149239111/a3a61c9ba14f%2C46013%2C1733149239111.1733149241428 2024-12-02T14:20:41,464 DEBUG [RS:0;a3a61c9ba14f:46013 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36835:36835)] 2024-12-02T14:20:41,505 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-12-02T14:20:41,506 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4 2024-12-02T14:20:41,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37975 is added to blk_1073741833_1009 (size=32) 2024-12-02T14:20:41,527 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T14:20:41,540 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T14:20:41,543 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T14:20:41,544 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,545 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T14:20:41,546 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T14:20:41,549 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T14:20:41,549 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,551 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T14:20:41,551 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T14:20:41,555 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T14:20:41,556 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,557 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T14:20:41,558 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T14:20:41,562 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T14:20:41,562 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:41,570 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T14:20:41,571 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T14:20:41,573 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/meta/1588230740 2024-12-02T14:20:41,575 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/meta/1588230740 2024-12-02T14:20:41,580 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T14:20:41,580 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T14:20:41,582 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T14:20:41,588 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T14:20:41,594 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T14:20:41,596 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68450769, jitterRate=0.01999594271183014}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T14:20:41,599 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1733149241528Initializing all the Stores at 1733149241530 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733149241530Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733149241540 (+10 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149241540Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733149241540Cleaning up temporary data from old regions at 1733149241580 (+40 ms)Region opened successfully at 1733149241598 (+18 ms) 2024-12-02T14:20:41,599 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T14:20:41,599 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T14:20:41,599 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T14:20:41,600 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T14:20:41,600 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T14:20:41,606 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T14:20:41,606 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733149241599Disabling compacts and flushes for region at 1733149241599Disabling writes for close at 1733149241600 (+1 ms)Writing region close event to WAL at 1733149241605 (+5 ms)Closed at 1733149241606 (+1 ms) 2024-12-02T14:20:41,610 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T14:20:41,610 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-12-02T14:20:41,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-02T14:20:41,634 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T14:20:41,637 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-02T14:20:41,791 DEBUG [a3a61c9ba14f:36883 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-02T14:20:41,806 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=a3a61c9ba14f,46013,1733149239111 2024-12-02T14:20:41,820 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a3a61c9ba14f,46013,1733149239111, state=OPENING 2024-12-02T14:20:41,824 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-02T14:20:41,825 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46013-0x1009b4dfd050001, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:20:41,826 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:20:41,827 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T14:20:41,829 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T14:20:41,833 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-02T14:20:41,836 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=a3a61c9ba14f,46013,1733149239111}] 2024-12-02T14:20:42,021 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-02T14:20:42,025 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45699, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-02T14:20:42,037 INFO [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-12-02T14:20:42,037 INFO [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T14:20:42,038 INFO [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-02T14:20:42,043 INFO [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a3a61c9ba14f%2C46013%2C1733149239111.meta, suffix=.meta, logDir=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/WALs/a3a61c9ba14f,46013,1733149239111, archiveDir=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/oldWALs, maxLogs=32 2024-12-02T14:20:42,060 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/WALs/a3a61c9ba14f,46013,1733149239111/a3a61c9ba14f%2C46013%2C1733149239111.meta.1733149242045.meta, exclude list is [], retry=0 2024-12-02T14:20:42,064 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37975,DS-a700cf0b-ff60-4712-a70e-6a39b5d11e1b,DISK] 2024-12-02T14:20:42,070 INFO [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/WALs/a3a61c9ba14f,46013,1733149239111/a3a61c9ba14f%2C46013%2C1733149239111.meta.1733149242045.meta 2024-12-02T14:20:42,074 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36835:36835)] 2024-12-02T14:20:42,074 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-02T14:20:42,076 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-02T14:20:42,080 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-02T14:20:42,087 INFO [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-02T14:20:42,096 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-02T14:20:42,097 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T14:20:42,097 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-12-02T14:20:42,097 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-12-02T14:20:42,105 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-02T14:20:42,109 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-02T14:20:42,109 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:42,112 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T14:20:42,112 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-12-02T14:20:42,114 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-12-02T14:20:42,114 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:42,115 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T14:20:42,115 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-02T14:20:42,116 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-02T14:20:42,116 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:42,117 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T14:20:42,117 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-02T14:20:42,119 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-02T14:20:42,119 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:42,120 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-02T14:20:42,120 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-12-02T14:20:42,121 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/meta/1588230740 2024-12-02T14:20:42,124 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/meta/1588230740 2024-12-02T14:20:42,126 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-12-02T14:20:42,126 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-12-02T14:20:42,127 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-02T14:20:42,130 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-12-02T14:20:42,132 INFO [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74554116, jitterRate=0.11094290018081665}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-02T14:20:42,132 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-12-02T14:20:42,134 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1733149242098Writing region info on filesystem at 1733149242098Initializing all the Stores at 1733149242104 (+6 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733149242104Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733149242105 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149242105Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1733149242105Cleaning up temporary data from old regions at 1733149242127 (+22 ms)Running coprocessor post-open hooks at 1733149242132 (+5 ms)Region opened successfully at 1733149242134 (+2 ms) 2024-12-02T14:20:42,140 INFO [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733149242012 2024-12-02T14:20:42,151 DEBUG [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-02T14:20:42,151 INFO [RS_OPEN_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-12-02T14:20:42,154 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=a3a61c9ba14f,46013,1733149239111 2024-12-02T14:20:42,157 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as a3a61c9ba14f,46013,1733149239111, state=OPEN 2024-12-02T14:20:42,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T14:20:42,160 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46013-0x1009b4dfd050001, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-02T14:20:42,160 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T14:20:42,160 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-02T14:20:42,161 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=a3a61c9ba14f,46013,1733149239111 2024-12-02T14:20:42,166 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-12-02T14:20:42,167 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=a3a61c9ba14f,46013,1733149239111 in 325 msec 2024-12-02T14:20:42,174 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-12-02T14:20:42,174 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 550 msec 2024-12-02T14:20:42,175 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-12-02T14:20:42,175 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-12-02T14:20:42,198 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:20:42,200 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46013,1733149239111, seqNum=-1] 2024-12-02T14:20:42,224 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:20:42,228 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52645, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:20:42,253 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.2810 sec 2024-12-02T14:20:42,253 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733149242253, completionTime=-1 2024-12-02T14:20:42,256 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-02T14:20:42,257 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-12-02T14:20:42,316 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-12-02T14:20:42,316 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733149302316 2024-12-02T14:20:42,316 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733149362316 2024-12-02T14:20:42,317 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 60 msec 2024-12-02T14:20:42,320 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3a61c9ba14f,36883,1733149238328-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-02T14:20:42,325 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3a61c9ba14f,36883,1733149238328-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T14:20:42,325 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3a61c9ba14f,36883,1733149238328-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T14:20:42,327 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-a3a61c9ba14f:36883, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T14:20:42,335 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-02T14:20:42,336 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-12-02T14:20:42,338 DEBUG [master/a3a61c9ba14f:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-12-02T14:20:42,364 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.072sec 2024-12-02T14:20:42,365 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-02T14:20:42,366 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-02T14:20:42,367 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-02T14:20:42,367 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-02T14:20:42,367 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-02T14:20:42,368 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3a61c9ba14f,36883,1733149238328-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-02T14:20:42,368 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3a61c9ba14f,36883,1733149238328-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-02T14:20:42,376 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-12-02T14:20:42,377 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-02T14:20:42,377 INFO [master/a3a61c9ba14f:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=a3a61c9ba14f,36883,1733149238328-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-02T14:20:42,442 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77a2a49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:42,444 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-02T14:20:42,444 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-02T14:20:42,447 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,36883,-1 for getting cluster id 2024-12-02T14:20:42,450 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:20:42,458 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5762d0f7-da11-4ede-9d35-feade635cfff' 2024-12-02T14:20:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:20:42,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5762d0f7-da11-4ede-9d35-feade635cfff" 2024-12-02T14:20:42,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7e1f4582, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:42,463 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,36883,-1] 2024-12-02T14:20:42,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:20:42,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:42,469 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:20:42,472 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c448862, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:42,472 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:20:42,481 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46013,1733149239111, seqNum=-1] 2024-12-02T14:20:42,482 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:20:42,484 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-02T14:20:42,503 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=a3a61c9ba14f,36883,1733149238328 2024-12-02T14:20:42,505 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43df6b1d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:42,505 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request a3a61c9ba14f,36883,-1 for getting cluster id 2024-12-02T14:20:42,505 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-12-02T14:20:42,507 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '5762d0f7-da11-4ede-9d35-feade635cfff' 2024-12-02T14:20:42,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-12-02T14:20:42,508 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "5762d0f7-da11-4ede-9d35-feade635cfff" 2024-12-02T14:20:42,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35c52ce8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:42,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [a3a61c9ba14f,36883,-1] 2024-12-02T14:20:42,509 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-12-02T14:20:42,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:20:42,513 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51452, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-12-02T14:20:42,515 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@674d2736, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-02T14:20:42,526 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[0] Thread=214, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=459, ProcessCount=11, AvailableMemoryMB=6226 2024-12-02T14:20:42,537 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-12-02T14:20:42,542 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is a3a61c9ba14f,36883,1733149238328 2024-12-02T14:20:42,545 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@79cb1a83 2024-12-02T14:20:42,546 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-02T14:20:42,549 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-02T14:20:42,550 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:20:42,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:20:42,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T14:20:42,565 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:42,570 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2a0af385 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@510e1100 2024-12-02T14:20:42,571 WARN [PEWorker-4 {}] client.ZKConnectionRegistry(87): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-12-02T14:20:42,576 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x2a0af385 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2a0af385 to 127.0.0.1:53873 2024-12-02T14:20:42,577 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:42,579 INFO [PEWorker-4 {}] master.HMaster(2490): Client=null/null create 'hbase:replication', {TABLE_ATTRIBUTES => {coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.regionserver.region.split_restriction.delimiter' => '-', 'hbase.regionserver.region.split_restriction.type' => 'DelimitedKeyPrefix'}}}, {NAME => 'hfileref', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'queue', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'sid', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-02T14:20:42,584 DEBUG [PEWorker-4 {}] procedure2.ProcedureExecutor(1139): Stored pid=5, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=hbase:replication 2024-12-02T14:20:42,590 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_PRE_OPERATION 2024-12-02T14:20:42,591 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:42,594 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-02T14:20:42,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37975 is added to blk_1073741835_1011 (size=1138) 2024-12-02T14:20:42,616 INFO [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 3edf40fb8fe820e5620b9e92d8c85956, NAME => 'hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:replication', {TABLE_ATTRIBUTES => {coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.regionserver.region.split_restriction.delimiter' => '-', 'hbase.regionserver.region.split_restriction.type' => 'DelimitedKeyPrefix', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'hfileref', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'queue', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'sid', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4 2024-12-02T14:20:42,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37975 is added to blk_1073741836_1012 (size=44) 2024-12-02T14:20:42,636 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(898): Instantiated hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T14:20:42,637 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1722): Closing 3edf40fb8fe820e5620b9e92d8c85956, disabling compactions & flushes 2024-12-02T14:20:42,637 INFO [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1755): Closing region hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956. 2024-12-02T14:20:42,637 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956. 2024-12-02T14:20:42,637 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956. after waiting 0 ms 2024-12-02T14:20:42,637 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956. 2024-12-02T14:20:42,637 INFO [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1973): Closed hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956. 2024-12-02T14:20:42,637 DEBUG [RegionOpenAndInit-hbase:replication-pool-0 {}] regionserver.HRegion(1676): Region close journal for 3edf40fb8fe820e5620b9e92d8c85956: Waiting for close lock at 1733149242637Disabling compacts and flushes for region at 1733149242637Disabling writes for close at 1733149242637Writing region close event to WAL at 1733149242637Closed at 1733149242637 2024-12-02T14:20:42,639 DEBUG [PEWorker-4 {}] procedure.ProcedureSyncWait(219): waitFor Creating table hbase:replication 2024-12-02T14:20:42,641 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_ADD_TO_META 2024-12-02T14:20:42,647 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956.","families":{"info":[{"qualifier":"regioninfo","vlen":43,"tag":[],"timestamp":"1733149242641"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733149242641"}]},"ts":"1733149242641"} 2024-12-02T14:20:42,654 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-12-02T14:20:42,657 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-02T14:20:42,661 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:replication","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733149242658"}]},"ts":"1733149242658"} 2024-12-02T14:20:42,669 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:replication, state=ENABLING in hbase:meta 2024-12-02T14:20:42,672 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:replication, region=3edf40fb8fe820e5620b9e92d8c85956, ASSIGN}] 2024-12-02T14:20:42,675 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=6, ppid=5, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:replication, region=3edf40fb8fe820e5620b9e92d8c85956, ASSIGN 2024-12-02T14:20:42,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T14:20:42,680 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=6, ppid=5, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:replication, region=3edf40fb8fe820e5620b9e92d8c85956, ASSIGN; state=OFFLINE, location=a3a61c9ba14f,46013,1733149239111; forceNewPlan=false, retain=false 2024-12-02T14:20:42,831 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=3edf40fb8fe820e5620b9e92d8c85956, regionState=OPENING, regionLocation=a3a61c9ba14f,46013,1733149239111 2024-12-02T14:20:42,836 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:replication, region=3edf40fb8fe820e5620b9e92d8c85956, ASSIGN because future has completed 2024-12-02T14:20:42,837 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=7, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3edf40fb8fe820e5620b9e92d8c85956, server=a3a61c9ba14f,46013,1733149239111}] 2024-12-02T14:20:42,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T14:20:42,999 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] handler.AssignRegionHandler(132): Open hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956. 2024-12-02T14:20:42,999 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-02T14:20:43,000 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] monitor.StreamSlowMonitor(122): New stream slow monitor rep 2024-12-02T14:20:43,003 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=a3a61c9ba14f%2C46013%2C1733149239111.rep, suffix=, logDir=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/WALs/a3a61c9ba14f,46013,1733149239111, archiveDir=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/oldWALs, maxLogs=32 2024-12-02T14:20:43,024 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/WALs/a3a61c9ba14f,46013,1733149239111/a3a61c9ba14f%2C46013%2C1733149239111.rep.1733149243005, exclude list is [], retry=0 2024-12-02T14:20:43,028 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37975,DS-a700cf0b-ff60-4712-a70e-6a39b5d11e1b,DISK] 2024-12-02T14:20:43,031 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/WALs/a3a61c9ba14f,46013,1733149239111/a3a61c9ba14f%2C46013%2C1733149239111.rep.1733149243005 2024-12-02T14:20:43,032 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36835:36835)] 2024-12-02T14:20:43,032 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(7752): Opening region: {ENCODED => 3edf40fb8fe820e5620b9e92d8c85956, NAME => 'hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956.', STARTKEY => '', ENDKEY => ''} 2024-12-02T14:20:43,032 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-02T14:20:43,032 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956. service=MultiRowMutationService 2024-12-02T14:20:43,033 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:replication successfully. 2024-12-02T14:20:43,033 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table replication 3edf40fb8fe820e5620b9e92d8c85956 2024-12-02T14:20:43,033 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(898): Instantiated hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-02T14:20:43,033 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(7794): checking encryption for 3edf40fb8fe820e5620b9e92d8c85956 2024-12-02T14:20:43,033 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(7797): checking classloading for 3edf40fb8fe820e5620b9e92d8c85956 2024-12-02T14:20:43,036 INFO [StoreOpener-3edf40fb8fe820e5620b9e92d8c85956-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family hfileref of region 3edf40fb8fe820e5620b9e92d8c85956 2024-12-02T14:20:43,038 INFO [StoreOpener-3edf40fb8fe820e5620b9e92d8c85956-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3edf40fb8fe820e5620b9e92d8c85956 columnFamilyName hfileref 2024-12-02T14:20:43,038 DEBUG [StoreOpener-3edf40fb8fe820e5620b9e92d8c85956-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:43,039 INFO [StoreOpener-3edf40fb8fe820e5620b9e92d8c85956-1 {}] regionserver.HStore(327): Store=3edf40fb8fe820e5620b9e92d8c85956/hfileref, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:20:43,040 INFO [StoreOpener-3edf40fb8fe820e5620b9e92d8c85956-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family queue of region 3edf40fb8fe820e5620b9e92d8c85956 2024-12-02T14:20:43,041 INFO [StoreOpener-3edf40fb8fe820e5620b9e92d8c85956-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3edf40fb8fe820e5620b9e92d8c85956 columnFamilyName queue 2024-12-02T14:20:43,042 DEBUG [StoreOpener-3edf40fb8fe820e5620b9e92d8c85956-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:43,042 INFO [StoreOpener-3edf40fb8fe820e5620b9e92d8c85956-1 {}] regionserver.HStore(327): Store=3edf40fb8fe820e5620b9e92d8c85956/queue, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:20:43,042 INFO [StoreOpener-3edf40fb8fe820e5620b9e92d8c85956-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family sid of region 3edf40fb8fe820e5620b9e92d8c85956 2024-12-02T14:20:43,045 INFO [StoreOpener-3edf40fb8fe820e5620b9e92d8c85956-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 3edf40fb8fe820e5620b9e92d8c85956 columnFamilyName sid 2024-12-02T14:20:43,045 DEBUG [StoreOpener-3edf40fb8fe820e5620b9e92d8c85956-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-02T14:20:43,046 INFO [StoreOpener-3edf40fb8fe820e5620b9e92d8c85956-1 {}] regionserver.HStore(327): Store=3edf40fb8fe820e5620b9e92d8c85956/sid, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-02T14:20:43,046 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1038): replaying wal for 3edf40fb8fe820e5620b9e92d8c85956 2024-12-02T14:20:43,048 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/replication/3edf40fb8fe820e5620b9e92d8c85956 2024-12-02T14:20:43,049 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/replication/3edf40fb8fe820e5620b9e92d8c85956 2024-12-02T14:20:43,052 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1048): stopping wal replay for 3edf40fb8fe820e5620b9e92d8c85956 2024-12-02T14:20:43,052 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1060): Cleaning up temporary data for 3edf40fb8fe820e5620b9e92d8c85956 2024-12-02T14:20:43,054 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:replication descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-02T14:20:43,057 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1093): writing seq id for 3edf40fb8fe820e5620b9e92d8c85956 2024-12-02T14:20:43,062 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/replication/3edf40fb8fe820e5620b9e92d8c85956/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-02T14:20:43,064 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1114): Opened 3edf40fb8fe820e5620b9e92d8c85956; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60541063, jitterRate=-0.09786786139011383}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-02T14:20:43,064 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 3edf40fb8fe820e5620b9e92d8c85956 2024-12-02T14:20:43,065 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegion(1006): Region open journal for 3edf40fb8fe820e5620b9e92d8c85956: Running coprocessor pre-open hook at 1733149243033Writing region info on filesystem at 1733149243034 (+1 ms)Initializing all the Stores at 1733149243035 (+1 ms)Instantiating store for column family {NAME => 'hfileref', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149243035Instantiating store for column family {NAME => 'queue', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149243036 (+1 ms)Instantiating store for column family {NAME => 'sid', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1733149243036Cleaning up temporary data from old regions at 1733149243052 (+16 ms)Running coprocessor post-open hooks at 1733149243064 (+12 ms)Region opened successfully at 1733149243065 (+1 ms) 2024-12-02T14:20:43,067 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956., pid=7, masterSystemTime=1733149242992 2024-12-02T14:20:43,071 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956. 2024-12-02T14:20:43,072 INFO [RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=7}] handler.AssignRegionHandler(153): Opened hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956. 2024-12-02T14:20:43,073 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=6 updating hbase:meta row=3edf40fb8fe820e5620b9e92d8c85956, regionState=OPEN, openSeqNum=2, regionLocation=a3a61c9ba14f,46013,1733149239111 2024-12-02T14:20:43,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=7, ppid=6, state=RUNNABLE, hasLock=false; OpenRegionProcedure 3edf40fb8fe820e5620b9e92d8c85956, server=a3a61c9ba14f,46013,1733149239111 because future has completed 2024-12-02T14:20:43,085 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=7, resume processing ppid=6 2024-12-02T14:20:43,087 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, ppid=6, state=SUCCESS, hasLock=false; OpenRegionProcedure 3edf40fb8fe820e5620b9e92d8c85956, server=a3a61c9ba14f,46013,1733149239111 in 242 msec 2024-12-02T14:20:43,092 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-12-02T14:20:43,092 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:replication, region=3edf40fb8fe820e5620b9e92d8c85956, ASSIGN in 413 msec 2024-12-02T14:20:43,096 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-02T14:20:43,097 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"hbase:replication","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733149243096"}]},"ts":"1733149243096"} 2024-12-02T14:20:43,101 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=hbase:replication, state=ENABLED in hbase:meta 2024-12-02T14:20:43,104 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=5, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=hbase:replication execute state=CREATE_TABLE_POST_OPERATION 2024-12-02T14:20:43,109 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, state=SUCCESS, hasLock=false; CreateTableProcedure table=hbase:replication in 525 msec 2024-12-02T14:20:43,156 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:replication', row='1-', locateType=CURRENT is [region=hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956., hostname=a3a61c9ba14f,46013,1733149239111, seqNum=2] 2024-12-02T14:20:43,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-12-02T14:20:43,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:20:43,176 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46013-0x1009b4dfd050001, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:20:43,189 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=4, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:43,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T14:20:43,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=8 2024-12-02T14:20:43,358 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=8}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-02T14:20:43,412 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=8}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:43,415 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=8}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-12-02T14:20:43,415 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=8}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=a3a61c9ba14f,46013,1733149239111, seqNum=-1] 2024-12-02T14:20:43,415 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=8}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-02T14:20:43,419 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52177, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=ClientService 2024-12-02T14:20:43,423 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:replication', row='1-a3a61c9ba14f,46013,1733149239111', locateType=CURRENT is [region=hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956., hostname=a3a61c9ba14f,46013,1733149239111, seqNum=2] 2024-12-02T14:20:43,431 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-12-02T14:20:43,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-12-02T14:20:43,438 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:43,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=4 2024-12-02T14:20:43,443 INFO [PEWorker-2 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-02T14:20:43,443 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=4, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 249 msec 2024-12-02T14:20:43,447 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 891 msec 2024-12-02T14:20:43,457 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:43,461 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4d9fc2b9 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3c642a73 2024-12-02T14:20:43,474 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:43,474 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-12-02T14:20:43,710 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:20:43,713 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-02T14:20:43,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:20:43,721 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T14:20:43,723 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x38bc94eb to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@55cc0562 2024-12-02T14:20:43,726 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x38bc94eb {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x38bc94eb to 127.0.0.1:53873 2024-12-02T14:20:43,726 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:43,731 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:43,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T14:20:43,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=10 2024-12-02T14:20:43,888 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:20:43,890 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:20:43,890 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:20:43,890 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:43,890 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:43,892 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2745ce79 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@30c2acff 2024-12-02T14:20:43,892 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:43,912 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:43,913 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:20:43,913 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-02T14:20:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-12-02T14:20:43,919 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:43,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-12-02T14:20:43,925 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-02T14:20:43,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 188 msec 2024-12-02T14:20:43,930 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 212 msec 2024-12-02T14:20:43,931 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:43,933 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0e1c543d to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2457d7ff 2024-12-02T14:20:43,937 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:43,937 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:44,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-12-02T14:20:44,039 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:20:44,042 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:44,044 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:44,046 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:44,049 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-02T14:20:44,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:20:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T14:20:44,054 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:44,056 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7a4a5b67 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7a493a17 2024-12-02T14:20:44,058 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x7a4a5b67 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7a4a5b67 to 127.0.0.1:53873 2024-12-02T14:20:44,059 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:44,064 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:44,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T14:20:44,218 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=12 2024-12-02T14:20:44,219 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:20:44,220 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:20:44,220 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:20:44,220 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:44,220 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:44,223 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x55ef2471 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@26508928 2024-12-02T14:20:44,223 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:44,249 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:44,249 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:20:44,249 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-02T14:20:44,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-12-02T14:20:44,254 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:44,258 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-12-02T14:20:44,259 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-02T14:20:44,259 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 190 msec 2024-12-02T14:20:44,261 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 210 msec 2024-12-02T14:20:44,270 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:44,272 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x50579f92 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@346c741b 2024-12-02T14:20:44,275 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:44,276 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:44,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-12-02T14:20:44,369 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:20:44,370 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:44,372 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:44,373 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:44,375 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:44,377 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-02T14:20:44,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:20:44,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T14:20:44,385 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:44,386 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x34204ff7 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2c3f8078 2024-12-02T14:20:44,395 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x34204ff7 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x34204ff7 to 127.0.0.1:53873 2024-12-02T14:20:44,395 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:44,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:44,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T14:20:44,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=14 2024-12-02T14:20:44,555 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:20:44,556 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:20:44,556 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:20:44,556 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:44,557 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:44,559 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x58adb9ef to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@33a23b05 2024-12-02T14:20:44,560 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:44,579 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:44,579 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:20:44,579 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-02T14:20:44,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-12-02T14:20:44,584 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:44,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-12-02T14:20:44,589 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-02T14:20:44,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 185 msec 2024-12-02T14:20:44,592 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 212 msec 2024-12-02T14:20:44,596 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:44,598 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0189aa18 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@164b087b 2024-12-02T14:20:44,602 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:44,602 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:44,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-12-02T14:20:44,699 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:20:44,699 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:44,701 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:44,702 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:44,703 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-02T14:20:44,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:20:44,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T14:20:44,707 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:44,709 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x047c5654 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4765c390 2024-12-02T14:20:44,717 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x047c5654 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x047c5654 to 127.0.0.1:53873 2024-12-02T14:20:44,717 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:44,722 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:44,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T14:20:44,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=16 2024-12-02T14:20:44,876 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:20:44,878 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:20:44,878 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:20:44,878 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:44,878 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:44,880 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6949737a to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@32a079c5 2024-12-02T14:20:44,881 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:44,904 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:44,905 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:20:44,905 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-02T14:20:44,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-12-02T14:20:44,910 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:44,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-12-02T14:20:44,914 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-02T14:20:44,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 188 msec 2024-12-02T14:20:44,917 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 211 msec 2024-12-02T14:20:44,922 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:44,924 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x131bc930 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1672972 2024-12-02T14:20:44,928 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:44,928 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:45,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-12-02T14:20:45,028 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:20:45,029 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:45,030 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:45,031 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-02T14:20:45,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:20:45,035 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:45,036 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1130a6fa to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@373932b6 2024-12-02T14:20:45,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T14:20:45,040 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x1130a6fa {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1130a6fa to 127.0.0.1:53873 2024-12-02T14:20:45,041 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:45,045 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:45,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T14:20:45,199 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=18 2024-12-02T14:20:45,199 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:20:45,201 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:20:45,201 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:20:45,201 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:45,201 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:45,203 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x23ee7e1d to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@11363c96 2024-12-02T14:20:45,206 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:45,232 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:45,232 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:20:45,233 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-02T14:20:45,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-12-02T14:20:45,238 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:45,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-12-02T14:20:45,246 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-02T14:20:45,246 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 193 msec 2024-12-02T14:20:45,249 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 214 msec 2024-12-02T14:20:45,255 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:45,257 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1b2757ae to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@576b0735 2024-12-02T14:20:45,261 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:45,261 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:45,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-12-02T14:20:45,359 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:20:45,360 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:45,361 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-02T14:20:45,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:20:45,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T14:20:45,366 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:45,367 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3d746a30 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7b5a75a6 2024-12-02T14:20:45,370 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x3d746a30 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3d746a30 to 127.0.0.1:53873 2024-12-02T14:20:45,370 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:45,374 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:45,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T14:20:45,528 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=20 2024-12-02T14:20:45,528 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:20:45,529 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:20:45,529 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:20:45,529 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:45,530 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:45,531 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x07fc8eb2 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@68e35a51 2024-12-02T14:20:45,531 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:45,557 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:45,557 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:20:45,557 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-02T14:20:45,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-12-02T14:20:45,562 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:45,565 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-12-02T14:20:45,566 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-02T14:20:45,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 188 msec 2024-12-02T14:20:45,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 205 msec 2024-12-02T14:20:45,573 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:45,575 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x38212336 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@47b80c71 2024-12-02T14:20:45,578 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:45,578 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:45,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-12-02T14:20:45,679 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:20:45,680 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:45,683 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:20:45,683 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:20:45,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=22, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:45,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-02T14:20:45,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:45,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T14:20:45,697 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:20:45,698 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-02T14:20:45,702 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:20:45,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:20:45,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:20:45,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-02T14:20:45,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:20:45,709 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:20:45,714 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:45,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-02T14:20:45,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T14:20:45,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=23 2024-12-02T14:20:45,871 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-02T14:20:45,871 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-02T14:20:45,871 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Replication stream was removed by a user 2024-12-02T14:20:45,871 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:45,871 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:45,873 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0c71eac2 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7d9dab3c 2024-12-02T14:20:45,881 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:45,882 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.ReplicationSourceManager(562): Done with the queue 1-a3a61c9ba14f,46013,1733149239111 2024-12-02T14:20:45,891 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=23}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=23 2024-12-02T14:20:45,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=23 2024-12-02T14:20:45,895 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:45,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-12-02T14:20:45,901 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-02T14:20:45,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 181 msec 2024-12-02T14:20:45,903 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-02T14:20:45,928 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-02T14:20:45,932 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 245 msec 2024-12-02T14:20:45,942 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:45,952 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=21, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:45,956 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=21, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=270 msec 2024-12-02T14:20:45,956 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:46,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=22 2024-12-02T14:20:46,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-12-02T14:20:46,009 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:20:46,010 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-02T14:20:46,011 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:20:46,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=24, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:46,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-12-02T14:20:46,016 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:46,019 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=24, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:46,021 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=24, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=8 msec 2024-12-02T14:20:46,022 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:46,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=24 2024-12-02T14:20:46,129 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:20:46,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'hbase:replication', row='', locateType=CURRENT is [region=hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956., hostname=a3a61c9ba14f,46013,1733149239111, seqNum=2] 2024-12-02T14:20:46,184 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[0] Thread=267 (was 214) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x38212336-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x50579f92 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0c71eac2-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x4d9fc2b9-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x38212336 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x6949737a-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: PacketResponder: BP-1636995057-172.17.0.2-1733149234894:blk_1073741837_1013, type=LAST_IN_PIPELINE java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.lang.Object.wait(Object.java:338) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x6949737a-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x07fc8eb2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x55ef2471 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0c71eac2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x23ee7e1d-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0189aa18 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0189aa18-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x58adb9ef java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x2745ce79-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x23ee7e1d java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x58adb9ef-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1064107970_22 at /127.0.0.1:47342 [Waiting for operation #6] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0e1c543d-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1064107970_22 at /127.0.0.1:47468 [Receiving block BP-1636995057-172.17.0.2-1733149234894:blk_1073741837_1013] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0c71eac2-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x07fc8eb2-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x131bc930-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x23ee7e1d-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x131bc930-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: AsyncFSWAL-0-hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4-prefix:a3a61c9ba14f,46013,1733149239111.rep java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x38212336-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x131bc930 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x55ef2471-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x2745ce79 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x58adb9ef-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x07fc8eb2-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x1b2757ae-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0e1c543d-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x50579f92-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0e1c543d java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x1b2757ae-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0189aa18-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x2745ce79-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x4d9fc2b9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x4d9fc2b9-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x50579f92-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x55ef2471-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x6949737a java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x1b2757ae java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_PRIORITY_REGION-regionserver/a3a61c9ba14f:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=509 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=542 (was 459) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=6167 (was 6226) 2024-12-02T14:20:46,197 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[0] Thread=267, OpenFileDescriptor=509, MaxFileDescriptor=1048576, SystemLoadAverage=542, ProcessCount=11, AvailableMemoryMB=6167 2024-12-02T14:20:46,199 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:20:46,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:20:46,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T14:20:46,204 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:46,206 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x50149e9b to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1177b368 2024-12-02T14:20:46,222 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x50149e9b {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x50149e9b to 127.0.0.1:53873 2024-12-02T14:20:46,222 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:46,246 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:46,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T14:20:46,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=26 2024-12-02T14:20:46,404 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=26}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-02T14:20:46,458 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=26}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:46,489 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-12-02T14:20:46,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-12-02T14:20:46,498 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:46,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-12-02T14:20:46,502 INFO [PEWorker-1 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-02T14:20:46,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 252 msec 2024-12-02T14:20:46,506 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 303 msec 2024-12-02T14:20:46,509 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:46,511 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6b4ad67c to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@245ac58 2024-12-02T14:20:46,518 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:46,518 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:46,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-12-02T14:20:46,519 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:20:46,520 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-02T14:20:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:20:46,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T14:20:46,525 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:46,527 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x42a10f43 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@13ab9b73 2024-12-02T14:20:46,542 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x42a10f43 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x42a10f43 to 127.0.0.1:53873 2024-12-02T14:20:46,542 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:46,549 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:46,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T14:20:46,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=28 2024-12-02T14:20:46,705 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:20:46,707 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:20:46,707 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:20:46,707 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:46,707 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:46,709 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7f2a44eb to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@58b0eb8d 2024-12-02T14:20:46,710 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:46,734 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:46,735 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:20:46,736 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-12-02T14:20:46,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-12-02T14:20:46,741 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:46,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-12-02T14:20:46,745 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-02T14:20:46,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 192 msec 2024-12-02T14:20:46,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 225 msec 2024-12-02T14:20:46,761 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:46,763 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x795c6f43 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1a0247f4 2024-12-02T14:20:46,767 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:46,767 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:46,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-12-02T14:20:46,848 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:20:46,850 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-02T14:20:46,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:20:46,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-02T14:20:46,859 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:46,860 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4fa79766 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2222ce26 2024-12-02T14:20:46,864 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x4fa79766 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4fa79766 to 127.0.0.1:53873 2024-12-02T14:20:46,864 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:46,875 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:46,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-02T14:20:47,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=30 2024-12-02T14:20:47,030 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:20:47,031 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:20:47,031 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:20:47,031 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:47,032 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:47,034 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5892bdc5 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4471224f 2024-12-02T14:20:47,034 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:47,051 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:47,051 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:20:47,051 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-12-02T14:20:47,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=30 2024-12-02T14:20:47,057 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:47,063 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-12-02T14:20:47,063 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-02T14:20:47,064 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 182 msec 2024-12-02T14:20:47,067 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 214 msec 2024-12-02T14:20:47,072 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:47,073 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x60782e1a to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2e37c25b 2024-12-02T14:20:47,076 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:47,076 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:47,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-12-02T14:20:47,168 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:20:47,170 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns1:testNamespacesAndTableCfsConfigConflict1=null},bandwidth=0,serial=false 2024-12-02T14:20:47,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:20:47,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-02T14:20:47,173 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:47,175 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7e405a8a to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2225f548 2024-12-02T14:20:47,178 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x7e405a8a {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7e405a8a to 127.0.0.1:53873 2024-12-02T14:20:47,178 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:47,179 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:47,181 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=31, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:47,183 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=31, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=11 msec 2024-12-02T14:20:47,183 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:47,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-12-02T14:20:47,279 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config 2024-12-02T14:20:47,280 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-02T14:20:47,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:20:47,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-02T14:20:47,284 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:47,285 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5cafe28e to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@81c36f3 2024-12-02T14:20:47,289 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x5cafe28e {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5cafe28e to 127.0.0.1:53873 2024-12-02T14:20:47,289 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:47,293 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:47,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-02T14:20:47,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=33 2024-12-02T14:20:47,448 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:20:47,450 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:20:47,450 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:20:47,450 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:47,450 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:47,452 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3c2d4426 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3fddc3b1 2024-12-02T14:20:47,453 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:47,482 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:47,483 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:20:47,483 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=33}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=33 2024-12-02T14:20:47,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=33 2024-12-02T14:20:47,488 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:47,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-12-02T14:20:47,496 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-02T14:20:47,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 195 msec 2024-12-02T14:20:47,500 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 216 msec 2024-12-02T14:20:47,507 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:47,508 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x68da7d20 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@251c5865 2024-12-02T14:20:47,515 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:47,515 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:47,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-12-02T14:20:47,599 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:20:47,600 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-02T14:20:47,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:20:47,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-02T14:20:47,604 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:47,606 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T14:20:47,606 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x695e91ff to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1cc87c74 2024-12-02T14:20:47,610 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x695e91ff {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x695e91ff to 127.0.0.1:53873 2024-12-02T14:20:47,610 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:47,611 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:47,613 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=34, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:47,615 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=34, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=13 msec 2024-12-02T14:20:47,615 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:47,645 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:replication' 2024-12-02T14:20:47,647 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-02T14:20:47,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-12-02T14:20:47,710 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config 2024-12-02T14:20:47,711 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:20:47,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:47,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-02T14:20:47,716 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:20:47,717 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-02T14:20:47,717 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:20:47,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:20:47,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:20:47,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-02T14:20:47,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:20:47,720 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:20:47,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:47,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-02T14:20:47,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=36 2024-12-02T14:20:47,878 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-02T14:20:47,878 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-02T14:20:47,878 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Replication stream was removed by a user 2024-12-02T14:20:47,878 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:47,878 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:47,880 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x73b3edb8 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3594a3b0 2024-12-02T14:20:47,880 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:47,881 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.ReplicationSourceManager(562): Done with the queue 1-a3a61c9ba14f,46013,1733149239111 2024-12-02T14:20:47,884 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=36}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=36 2024-12-02T14:20:47,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=36 2024-12-02T14:20:47,887 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:47,891 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-12-02T14:20:47,891 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 165 msec 2024-12-02T14:20:47,891 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-02T14:20:47,891 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-02T14:20:47,898 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-02T14:20:47,900 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 187 msec 2024-12-02T14:20:48,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-12-02T14:20:48,029 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:20:48,030 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:20:48,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:48,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-12-02T14:20:48,033 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:48,034 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=37, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:48,036 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=37, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-02T14:20:48,036 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:48,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-12-02T14:20:48,139 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-02T14:20:48,140 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:20:48,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:48,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-02T14:20:48,142 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:48,144 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=38, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:48,145 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=38, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-02T14:20:48,145 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:48,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-12-02T14:20:48,250 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:20:48,278 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[0] Thread=290 (was 267) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x5892bdc5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x68da7d20-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x6b4ad67c-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x60782e1a java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x68da7d20 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x68da7d20-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x3c2d4426-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x7f2a44eb-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x5892bdc5-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x73b3edb8-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x6b4ad67c java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x795c6f43 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x3c2d4426 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x60782e1a-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x73b3edb8-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x795c6f43-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x3c2d4426-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x7f2a44eb java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x795c6f43-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x60782e1a-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x5892bdc5-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x7f2a44eb-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x6b4ad67c-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x73b3edb8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=541 (was 509) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=542 (was 542), ProcessCount=11 (was 11), AvailableMemoryMB=6147 (was 6167) 2024-12-02T14:20:48,290 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAddRemovePeer[0] Thread=290, OpenFileDescriptor=541, MaxFileDescriptor=1048576, SystemLoadAverage=542, ProcessCount=11, AvailableMemoryMB=6147 2024-12-02T14:20:48,291 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:20:48,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=39, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:20:48,295 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:48,297 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x35e5b0fc to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3fbc25c5 2024-12-02T14:20:48,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-02T14:20:48,306 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x35e5b0fc {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x35e5b0fc to 127.0.0.1:53873 2024-12-02T14:20:48,306 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:48,320 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:48,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-02T14:20:48,474 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=40 2024-12-02T14:20:48,474 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=40}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-02T14:20:48,513 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=40}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:48,516 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=40}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=40 2024-12-02T14:20:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=40 2024-12-02T14:20:48,519 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:48,525 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-12-02T14:20:48,525 INFO [PEWorker-3 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-02T14:20:48,525 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 200 msec 2024-12-02T14:20:48,528 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 234 msec 2024-12-02T14:20:48,532 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:48,534 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x566287a3 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7342f5e7 2024-12-02T14:20:48,537 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:48,537 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:48,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=39 2024-12-02T14:20:48,619 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:20:48,619 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:20:48,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=41, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:20:48,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=41 2024-12-02T14:20:48,623 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:48,624 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x306cc8a2 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@a3fa624 2024-12-02T14:20:48,629 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x306cc8a2 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x306cc8a2 to 127.0.0.1:53873 2024-12-02T14:20:48,630 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:48,630 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:177) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:48,631 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=41, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:177) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:48,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=41, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=13 msec 2024-12-02T14:20:48,633 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:48,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=41 2024-12-02T14:20:48,729 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Replication peer 1 already exists 2024-12-02T14:20:48,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-02T14:20:48,740 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:20:48,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:48,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T14:20:48,743 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:48,744 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=42, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:48,745 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-02T14:20:48,745 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-02T14:20:48,746 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=42, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-02T14:20:48,746 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:48,747 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_replication 2024-12-02T14:20:48,747 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_replication Metrics about Tables on a single HBase RegionServer 2024-12-02T14:20:48,748 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T14:20:48,748 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-02T14:20:48,749 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-02T14:20:48,749 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-02T14:20:48,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-12-02T14:20:48,849 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:20:48,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-02T14:20:48,852 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=2, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:20:48,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=43, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:20:48,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=43 2024-12-02T14:20:48,856 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test2' 2024-12-02T14:20:48,857 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1df7682d to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@67a228e7 2024-12-02T14:20:48,864 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x1df7682d {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1df7682d to 127.0.0.1:53873 2024-12-02T14:20:48,864 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test2 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:48,876 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=44, ppid=43, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:48,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=43 2024-12-02T14:20:49,032 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=44 2024-12-02T14:20:49,032 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=44}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=2, type=ADD_PEER 2024-12-02T14:20:49,075 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=44}] regionserver.ReplicationSource(231): queueId=2-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 2, currentBandwidth=0 2024-12-02T14:20:49,079 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=44}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=44 2024-12-02T14:20:49,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=44 2024-12-02T14:20:49,083 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 2 for ADD on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:49,087 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=44, resume processing ppid=43 2024-12-02T14:20:49,087 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, ppid=43, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 207 msec 2024-12-02T14:20:49,088 INFO [PEWorker-3 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 2, config clusterKey=hbase+zk://127.0.0.1:53873/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-02T14:20:49,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 236 msec 2024-12-02T14:20:49,096 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,2-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test2' 2024-12-02T14:20:49,097 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,2-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6ad64b23 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5c89ada0 2024-12-02T14:20:49,100 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,2-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:49,100 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,2-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=2, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:49,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=43 2024-12-02T14:20:49,169 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 2 completed 2024-12-02T14:20:49,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-02T14:20:49,170 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:20:49,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=45, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:49,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=45 2024-12-02T14:20:49,175 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:20:49,176 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-02T14:20:49,176 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:20:49,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:20:49,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:20:49,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-02T14:20:49,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:20:49,179 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:20:49,181 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:49,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=45 2024-12-02T14:20:49,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=46 2024-12-02T14:20:49,335 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-02T14:20:49,335 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-02T14:20:49,335 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Replication stream was removed by a user 2024-12-02T14:20:49,335 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:49,335 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:49,337 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2c5c90e0 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@468d6da2 2024-12-02T14:20:49,337 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:49,338 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.ReplicationSourceManager(562): Done with the queue 1-a3a61c9ba14f,46013,1733149239111 2024-12-02T14:20:49,341 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=46 2024-12-02T14:20:49,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=46 2024-12-02T14:20:49,344 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:49,349 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=45 2024-12-02T14:20:49,350 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-02T14:20:49,350 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=45, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 164 msec 2024-12-02T14:20:49,350 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-02T14:20:49,356 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-02T14:20:49,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 185 msec 2024-12-02T14:20:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=45 2024-12-02T14:20:49,489 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:20:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-02T14:20:49,490 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:20:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=47, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:49,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-02T14:20:49,496 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/sync-rep-state 2024-12-02T14:20:49,496 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/peer-state 2024-12-02T14:20:49,497 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/new-sync-rep-state 2024-12-02T14:20:49,499 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/new-sync-rep-state 2024-12-02T14:20:49,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/2 2024-12-02T14:20:49,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/peer-state 2024-12-02T14:20:49,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/sync-rep-state 2024-12-02T14:20:49,500 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2 2024-12-02T14:20:49,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-02T14:20:49,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=48 2024-12-02T14:20:49,656 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=2, type=REMOVE_PEER 2024-12-02T14:20:49,656 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 2: 0 2024-12-02T14:20:49,656 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSource(687): peerId=2, Closing source 2-a3a61c9ba14f,46013,1733149239111 because: Replication stream was removed by a user 2024-12-02T14:20:49,656 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,2-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=2, Interrupted while sleeping between retries 2024-12-02T14:20:49,656 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,2-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test2' 2024-12-02T14:20:49,657 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,2-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x69ca93d7 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@b4ca9f1 2024-12-02T14:20:49,657 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,2-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:49,658 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSourceManager(562): Done with the queue 2-a3a61c9ba14f,46013,1733149239111 2024-12-02T14:20:49,661 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=48 2024-12-02T14:20:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=48 2024-12-02T14:20:49,664 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 2 for REMOVE on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:49,667 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=48, resume processing ppid=47 2024-12-02T14:20:49,667 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 2, move on 2024-12-02T14:20:49,667 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, ppid=47, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 162 msec 2024-12-02T14:20:49,667 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 2, move on 2024-12-02T14:20:49,675 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(120): Successfully removed peer 2 2024-12-02T14:20:49,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 184 msec 2024-12-02T14:20:49,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=47 2024-12-02T14:20:49,809 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 2 completed 2024-12-02T14:20:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-02T14:20:49,810 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:20:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=49, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:49,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-12-02T14:20:49,813 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:49,814 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=49, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:49,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=49, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-02T14:20:49,816 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-12-02T14:20:49,919 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-02T14:20:49,920 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:20:49,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T14:20:49,923 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:49,924 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=50, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:49,925 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=50, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-02T14:20:49,925 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:50,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-12-02T14:20:50,029 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:20:50,046 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAddRemovePeer[0] Thread=302 (was 290) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x2c5c90e0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x69ca93d7-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x2c5c90e0-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x69ca93d7-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x566287a3-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x69ca93d7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x6ad64b23-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x566287a3-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x6ad64b23 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x6ad64b23-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x2c5c90e0-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x566287a3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=555 (was 541) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=542 (was 542), ProcessCount=11 (was 11), AvailableMemoryMB=6136 (was 6147) 2024-12-02T14:20:50,055 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[0] Thread=302, OpenFileDescriptor=555, MaxFileDescriptor=1048576, SystemLoadAverage=542, ProcessCount=11, AvailableMemoryMB=6135 2024-12-02T14:20:50,057 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=whatever,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:20:50,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=51, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:20:50,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-12-02T14:20:50,061 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:470) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:467) ~[classes/:?] ... 11 more 2024-12-02T14:20:50,063 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=51, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:470) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:467) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:50,065 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=51, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=6 msec 2024-12-02T14:20:50,065 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:50,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=51 2024-12-02T14:20:50,169 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Can not instantiate configured replication endpoint class=whatever 2024-12-02T14:20:50,173 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:20:50,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:50,176 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:50,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T14:20:50,177 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=52, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:50,179 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=52, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-02T14:20:50,179 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:50,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-12-02T14:20:50,288 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-02T14:20:50,289 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:20:50,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=53, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:50,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-02T14:20:50,292 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:50,294 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=53, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:50,295 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=53, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-02T14:20:50,295 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:50,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=53 2024-12-02T14:20:50,399 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:20:50,413 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[0] Thread=302 (was 302), OpenFileDescriptor=555 (was 555), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=542 (was 542), ProcessCount=11 (was 11), AvailableMemoryMB=6134 (was 6135) 2024-12-02T14:20:50,422 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[0] Thread=302, OpenFileDescriptor=555, MaxFileDescriptor=1048576, SystemLoadAverage=542, ProcessCount=11, AvailableMemoryMB=6134 2024-12-02T14:20:50,423 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=whatever,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:20:50,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:20:50,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T14:20:50,427 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] ... 12 more 2024-12-02T14:20:50,428 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=54, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:50,430 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=54, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=6 msec 2024-12-02T14:20:50,430 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:50,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-12-02T14:20:50,539 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Invalid cluster key: whatever 2024-12-02T14:20:50,540 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:20:50,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=55, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:50,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-02T14:20:50,543 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:50,544 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=55, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:50,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=55, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-02T14:20:50,545 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:50,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=55 2024-12-02T14:20:50,648 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-02T14:20:50,649 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:20:50,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:50,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T14:20:50,652 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:50,653 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=56, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:50,654 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=56, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-02T14:20:50,654 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:50,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-12-02T14:20:50,758 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:20:50,774 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[0] Thread=302 (was 302), OpenFileDescriptor=555 (was 555), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=542 (was 542), ProcessCount=11 (was 11), AvailableMemoryMB=6133 (was 6134) 2024-12-02T14:20:50,785 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[0] Thread=302, OpenFileDescriptor=555, MaxFileDescriptor=1048576, SystemLoadAverage=542, ProcessCount=11, AvailableMemoryMB=6133 2024-12-02T14:20:50,786 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:20:50,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=57, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:20:50,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-02T14:20:50,790 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:50,792 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x11d630fe to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1082626c 2024-12-02T14:20:50,796 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x11d630fe {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x11d630fe to 127.0.0.1:53873 2024-12-02T14:20:50,796 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:50,807 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=58, ppid=57, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:50,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-02T14:20:50,968 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=58 2024-12-02T14:20:50,968 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=58}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-02T14:20:50,998 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=58}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:51,001 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=58}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=58 2024-12-02T14:20:51,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=58 2024-12-02T14:20:51,004 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:51,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=58, resume processing ppid=57 2024-12-02T14:20:51,007 INFO [PEWorker-3 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-02T14:20:51,007 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, ppid=57, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 197 msec 2024-12-02T14:20:51,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 221 msec 2024-12-02T14:20:51,015 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:51,016 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0a6d0f3c to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@10ce19c4 2024-12-02T14:20:51,018 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:51,019 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:51,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=57 2024-12-02T14:20:51,108 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:20:51,109 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-02T14:20:51,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=59, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:20:51,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-02T14:20:51,112 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:51,114 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x203c96c7 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@73ca326 2024-12-02T14:20:51,116 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x203c96c7 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x203c96c7 to 127.0.0.1:53873 2024-12-02T14:20:51,116 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:51,120 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:51,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-02T14:20:51,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=60 2024-12-02T14:20:51,275 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:20:51,276 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:20:51,276 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:20:51,276 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:51,277 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:51,278 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x73b9b2fd to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@347c80ac 2024-12-02T14:20:51,279 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:51,303 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:51,303 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:20:51,303 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=60 2024-12-02T14:20:51,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=60 2024-12-02T14:20:51,308 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:51,312 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-12-02T14:20:51,312 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 189 msec 2024-12-02T14:20:51,312 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-02T14:20:51,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 203 msec 2024-12-02T14:20:51,319 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:51,320 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x73754ab7 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@be3b004 2024-12-02T14:20:51,325 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:51,325 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:51,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=59 2024-12-02T14:20:51,429 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:20:51,430 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-12-02T14:20:51,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=61, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:20:51,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-02T14:20:51,434 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:51,435 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x169cd4ac to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7b72d845 2024-12-02T14:20:51,439 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x169cd4ac {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x169cd4ac to 127.0.0.1:53873 2024-12-02T14:20:51,439 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:51,443 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-02T14:20:51,596 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=62 2024-12-02T14:20:51,597 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:20:51,598 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:20:51,598 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:20:51,598 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:51,598 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:51,600 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x78eb6469 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5f1d9e5f 2024-12-02T14:20:51,600 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:51,624 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:51,624 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:20:51,624 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-12-02T14:20:51,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=62 2024-12-02T14:20:51,627 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:51,632 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=62, resume processing ppid=61 2024-12-02T14:20:51,632 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, ppid=61, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 184 msec 2024-12-02T14:20:51,632 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-12-02T14:20:51,635 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 202 msec 2024-12-02T14:20:51,643 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:51,645 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4442dc41 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@cd6e326 2024-12-02T14:20:51,649 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:51,650 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=61 2024-12-02T14:20:51,749 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:20:51,750 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:51,751 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-02T14:20:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:20:51,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-02T14:20:51,755 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:51,756 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7a3c0618 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1eb8ebbf 2024-12-02T14:20:51,758 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x7a3c0618 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7a3c0618 to 127.0.0.1:53873 2024-12-02T14:20:51,758 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:51,762 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:51,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-02T14:20:51,915 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=64 2024-12-02T14:20:51,915 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:20:51,917 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:20:51,917 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:20:51,917 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:51,917 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:51,920 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x38bf5644 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@363c7e22 2024-12-02T14:20:51,920 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:51,944 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:51,945 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:20:51,961 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=64 2024-12-02T14:20:51,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=64 2024-12-02T14:20:51,966 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:51,971 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-12-02T14:20:51,971 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 205 msec 2024-12-02T14:20:51,971 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-02T14:20:51,976 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 220 msec 2024-12-02T14:20:51,993 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:51,995 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x08ebae32 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1ac5167d 2024-12-02T14:20:52,018 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:52,018 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:52,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-12-02T14:20:52,069 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:20:52,069 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:52,070 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:20:52,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=65, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:52,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-12-02T14:20:52,075 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:20:52,077 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-02T14:20:52,077 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:20:52,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:20:52,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:20:52,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-02T14:20:52,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:20:52,080 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:20:52,082 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-12-02T14:20:52,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=66 2024-12-02T14:20:52,236 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-02T14:20:52,236 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-02T14:20:52,236 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Replication stream was removed by a user 2024-12-02T14:20:52,236 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:52,236 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:52,238 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x493fbdaa to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@27abda70 2024-12-02T14:20:52,238 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:52,238 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSourceManager(562): Done with the queue 1-a3a61c9ba14f,46013,1733149239111 2024-12-02T14:20:52,241 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-12-02T14:20:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=66 2024-12-02T14:20:52,244 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:52,247 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=66, resume processing ppid=65 2024-12-02T14:20:52,247 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-02T14:20:52,247 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, ppid=65, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 163 msec 2024-12-02T14:20:52,247 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-02T14:20:52,254 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-02T14:20:52,257 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 184 msec 2024-12-02T14:20:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=65 2024-12-02T14:20:52,389 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:20:52,389 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:20:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-02T14:20:52,391 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:52,393 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=67, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:52,394 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=67, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-02T14:20:52,394 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-12-02T14:20:52,499 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-02T14:20:52,500 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:20:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T14:20:52,503 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:52,504 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=68, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:52,506 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=68, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-02T14:20:52,506 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:52,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-12-02T14:20:52,609 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:20:52,629 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[0] Thread=326 (was 302) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x73754ab7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x78eb6469 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x73754ab7-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x73b9b2fd-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x4442dc41-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x78eb6469-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x73b9b2fd java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x08ebae32-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x73b9b2fd-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x493fbdaa-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x08ebae32 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x493fbdaa-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x4442dc41 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x4442dc41-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x38bf5644 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0a6d0f3c-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0a6d0f3c-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x08ebae32-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x38bf5644-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0a6d0f3c java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x38bf5644-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x493fbdaa java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x73754ab7-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x78eb6469-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) - Thread LEAK? -, OpenFileDescriptor=587 (was 555) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=531 (was 542), ProcessCount=11 (was 11), AvailableMemoryMB=6091 (was 6133) 2024-12-02T14:20:52,640 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[0] Thread=326, OpenFileDescriptor=585, MaxFileDescriptor=1048576, SystemLoadAverage=531, ProcessCount=11, AvailableMemoryMB=6088 2024-12-02T14:20:52,642 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:20:52,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=69, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:20:52,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69 2024-12-02T14:20:52,645 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:52,646 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x021cbcd6 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4de47a88 2024-12-02T14:20:52,648 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x021cbcd6 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x021cbcd6 to 127.0.0.1:53873 2024-12-02T14:20:52,649 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:52,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:52,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69 2024-12-02T14:20:52,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=70 2024-12-02T14:20:52,812 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=70}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-02T14:20:52,854 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=70}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:52,857 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=70}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=70 2024-12-02T14:20:52,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=70 2024-12-02T14:20:52,860 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:52,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=70, resume processing ppid=69 2024-12-02T14:20:52,863 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=69, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 201 msec 2024-12-02T14:20:52,863 INFO [PEWorker-5 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-02T14:20:52,865 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 221 msec 2024-12-02T14:20:52,877 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:52,879 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x67c7b603 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5aa40011 2024-12-02T14:20:52,885 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:52,885 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:52,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=69 2024-12-02T14:20:52,958 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:20:52,959 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-02T14:20:52,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=71, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:20:52,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71 2024-12-02T14:20:52,962 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:52,963 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6cec597e to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4729ee6b 2024-12-02T14:20:52,967 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x6cec597e {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6cec597e to 127.0.0.1:53873 2024-12-02T14:20:52,967 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:52,972 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:53,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71 2024-12-02T14:20:53,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=72 2024-12-02T14:20:53,125 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:20:53,126 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:20:53,126 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:20:53,126 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:53,126 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:53,127 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2f43660d to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6f953ae1 2024-12-02T14:20:53,127 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:53,150 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:53,150 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:20:53,150 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=72 2024-12-02T14:20:53,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=72 2024-12-02T14:20:53,155 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:53,160 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=72, resume processing ppid=71 2024-12-02T14:20:53,160 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-02T14:20:53,160 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, ppid=71, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 183 msec 2024-12-02T14:20:53,162 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 201 msec 2024-12-02T14:20:53,169 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:53,170 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x41937176 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@28b25656 2024-12-02T14:20:53,173 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:53,174 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:53,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=71 2024-12-02T14:20:53,278 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:20:53,279 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:53,280 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-12-02T14:20:53,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=73, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:20:53,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-12-02T14:20:53,283 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:53,284 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6f52df89 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5bd11f5e 2024-12-02T14:20:53,290 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x6f52df89 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6f52df89 to 127.0.0.1:53873 2024-12-02T14:20:53,290 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:53,294 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:53,383 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T14:20:53,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-12-02T14:20:53,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=74 2024-12-02T14:20:53,447 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:20:53,448 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:20:53,448 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:20:53,449 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:53,449 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:53,450 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x21606365 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6d1de9ea 2024-12-02T14:20:53,451 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:53,473 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:53,473 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:20:53,473 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=74 2024-12-02T14:20:53,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=74 2024-12-02T14:20:53,476 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:53,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=73 2024-12-02T14:20:53,479 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-12-02T14:20:53,479 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=73, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 182 msec 2024-12-02T14:20:53,480 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 199 msec 2024-12-02T14:20:53,494 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:53,495 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x08285bc1 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@109cbc0d 2024-12-02T14:20:53,498 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:53,498 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:53,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-12-02T14:20:53,598 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:20:53,599 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:53,600 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:53,601 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-02T14:20:53,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=75, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:20:53,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-12-02T14:20:53,604 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:53,606 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1bfad761 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@915c61c 2024-12-02T14:20:53,609 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x1bfad761 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1bfad761 to 127.0.0.1:53873 2024-12-02T14:20:53,609 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:53,613 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:53,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-12-02T14:20:53,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=76 2024-12-02T14:20:53,767 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:20:53,769 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:20:53,770 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:20:53,770 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:53,770 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:53,773 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x60a85739 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3cf6e5b0 2024-12-02T14:20:53,773 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:53,801 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:53,801 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:20:53,801 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=76 2024-12-02T14:20:53,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=76 2024-12-02T14:20:53,804 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:53,807 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=76, resume processing ppid=75 2024-12-02T14:20:53,807 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-02T14:20:53,807 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, ppid=75, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 191 msec 2024-12-02T14:20:53,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 206 msec 2024-12-02T14:20:53,815 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:53,816 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6b082c32 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7402ec3b 2024-12-02T14:20:53,819 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:53,819 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:53,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=75 2024-12-02T14:20:53,918 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:20:53,919 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:53,920 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:53,922 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-02T14:20:53,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=77, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:20:53,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-02T14:20:53,925 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:53,926 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x078bae8e to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3bee5f6c 2024-12-02T14:20:53,928 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x078bae8e {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x078bae8e to 127.0.0.1:53873 2024-12-02T14:20:53,929 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:53,933 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:54,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-02T14:20:54,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=78 2024-12-02T14:20:54,087 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:20:54,091 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:20:54,091 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:20:54,091 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:54,092 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:54,098 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4350833a to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4775d203 2024-12-02T14:20:54,098 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:54,113 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:54,113 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:20:54,114 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=78 2024-12-02T14:20:54,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=78 2024-12-02T14:20:54,116 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:54,118 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-12-02T14:20:54,119 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 183 msec 2024-12-02T14:20:54,119 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-02T14:20:54,120 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 197 msec 2024-12-02T14:20:54,128 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:54,129 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x013b2599 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7a9e2740 2024-12-02T14:20:54,132 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:54,132 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:54,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=77 2024-12-02T14:20:54,239 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:20:54,240 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:54,242 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:54,245 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-02T14:20:54,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=79, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:20:54,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-12-02T14:20:54,249 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:54,250 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3de710aa to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1574f63 2024-12-02T14:20:54,252 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x3de710aa {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3de710aa to 127.0.0.1:53873 2024-12-02T14:20:54,252 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:54,256 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:54,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-12-02T14:20:54,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=80 2024-12-02T14:20:54,410 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:20:54,414 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:20:54,414 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:20:54,414 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:54,415 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:54,419 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7a8e2896 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1607f5a2 2024-12-02T14:20:54,420 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:54,440 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:54,441 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:20:54,441 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=80 2024-12-02T14:20:54,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=80 2024-12-02T14:20:54,444 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:54,447 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=79 2024-12-02T14:20:54,447 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-02T14:20:54,447 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=79, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 188 msec 2024-12-02T14:20:54,449 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 202 msec 2024-12-02T14:20:54,462 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:54,464 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x787b4835 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@311c1bca 2024-12-02T14:20:54,467 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:54,468 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:54,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=79 2024-12-02T14:20:54,570 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:20:54,571 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:54,574 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:54,576 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-02T14:20:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=81, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:20:54,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-02T14:20:54,579 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:54,581 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3f8d0f13 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@42f4a13e 2024-12-02T14:20:54,583 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x3f8d0f13 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3f8d0f13 to 127.0.0.1:53873 2024-12-02T14:20:54,584 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:54,588 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-02T14:20:54,742 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=82 2024-12-02T14:20:54,742 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:20:54,746 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:20:54,746 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:20:54,746 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:54,747 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:54,750 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x671a6cfd to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@43ee1f06 2024-12-02T14:20:54,751 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:54,771 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:54,771 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:20:54,771 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=82 2024-12-02T14:20:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=82 2024-12-02T14:20:54,774 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:54,776 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-12-02T14:20:54,776 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-02T14:20:54,777 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 187 msec 2024-12-02T14:20:54,778 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 201 msec 2024-12-02T14:20:54,784 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:54,785 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2b5480af to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@8445921 2024-12-02T14:20:54,788 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:54,788 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-12-02T14:20:54,899 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:20:54,900 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:54,901 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-02T14:20:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:20:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-02T14:20:54,905 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:54,907 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5903d7fb to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@218be816 2024-12-02T14:20:54,910 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x5903d7fb {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5903d7fb to 127.0.0.1:53873 2024-12-02T14:20:54,910 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:54,914 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:55,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-02T14:20:55,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=84 2024-12-02T14:20:55,068 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:20:55,073 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=84 2024-12-02T14:20:55,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=84 2024-12-02T14:20:55,077 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:55,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-12-02T14:20:55,080 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-02T14:20:55,080 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 163 msec 2024-12-02T14:20:55,082 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 178 msec 2024-12-02T14:20:55,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-12-02T14:20:55,220 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:20:55,221 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:55,224 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:55,227 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-12-02T14:20:55,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:20:55,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-02T14:20:55,231 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:55,233 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x790bbdc4 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@677569c8 2024-12-02T14:20:55,236 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x790bbdc4 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x790bbdc4 to 127.0.0.1:53873 2024-12-02T14:20:55,236 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:55,240 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:55,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-02T14:20:55,393 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=86 2024-12-02T14:20:55,393 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:20:55,395 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:20:55,395 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:20:55,396 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:55,396 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:55,398 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4251adba to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7e390fd1 2024-12-02T14:20:55,398 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:55,420 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:55,420 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:20:55,420 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=86 2024-12-02T14:20:55,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=86 2024-12-02T14:20:55,423 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:55,425 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-12-02T14:20:55,425 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-12-02T14:20:55,425 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 183 msec 2024-12-02T14:20:55,426 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 198 msec 2024-12-02T14:20:55,438 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:55,440 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4d41dcc7 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@58f6b530 2024-12-02T14:20:55,443 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:55,443 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:55,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-12-02T14:20:55,549 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:20:55,550 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:55,553 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-12-02T14:20:55,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:20:55,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-02T14:20:55,559 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:55,560 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3be377ab to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2b9db5a4 2024-12-02T14:20:55,563 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x3be377ab {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3be377ab to 127.0.0.1:53873 2024-12-02T14:20:55,563 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:55,567 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-02T14:20:55,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=88 2024-12-02T14:20:55,719 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:20:55,720 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:20:55,720 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:20:55,720 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:55,720 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:55,722 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x725ad586 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3076e2b9 2024-12-02T14:20:55,722 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:55,743 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:55,743 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:20:55,743 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-12-02T14:20:55,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-12-02T14:20:55,747 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:55,750 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-12-02T14:20:55,750 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 180 msec 2024-12-02T14:20:55,750 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-12-02T14:20:55,752 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 197 msec 2024-12-02T14:20:55,758 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:55,760 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0782edbf to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@e111f94 2024-12-02T14:20:55,762 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:55,762 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:55,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-12-02T14:20:55,879 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:20:55,880 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:55,882 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:20:55,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:55,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-02T14:20:55,891 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:20:55,892 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-02T14:20:55,893 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:20:55,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:20:55,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:20:55,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-02T14:20:55,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:20:55,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:20:55,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-02T14:20:56,051 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=90 2024-12-02T14:20:56,051 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-02T14:20:56,052 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-02T14:20:56,052 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Replication stream was removed by a user 2024-12-02T14:20:56,052 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:56,053 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:56,058 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0cacb689 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6644d0af 2024-12-02T14:20:56,059 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:56,059 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSourceManager(562): Done with the queue 1-a3a61c9ba14f,46013,1733149239111 2024-12-02T14:20:56,062 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-12-02T14:20:56,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-12-02T14:20:56,065 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:56,066 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-12-02T14:20:56,067 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 168 msec 2024-12-02T14:20:56,067 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-02T14:20:56,067 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-02T14:20:56,072 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-02T14:20:56,074 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 190 msec 2024-12-02T14:20:56,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-12-02T14:20:56,210 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:20:56,211 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:20:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-02T14:20:56,215 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:56,216 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=91, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:56,217 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=91, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-02T14:20:56,217 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:56,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-12-02T14:20:56,322 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-02T14:20:56,324 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:20:56,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=92, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:56,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-02T14:20:56,327 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:56,328 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=92, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:56,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=92, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-02T14:20:56,329 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=92 2024-12-02T14:20:56,439 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:20:56,464 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[0] Thread=377 (was 326) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x67c7b603-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x67c7b603 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x013b2599 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x7a8e2896-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x21606365 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x6b082c32-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x08285bc1-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x4350833a-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x4251adba-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x41937176-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x2b5480af java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0782edbf-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x2b5480af-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x725ad586 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x4251adba-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x787b4835 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x21606365-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x4350833a java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x41937176-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0cacb689 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x60a85739-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x4d41dcc7-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x787b4835-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x6b082c32-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x7a8e2896-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x013b2599-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x671a6cfd-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x671a6cfd-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0cacb689-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x4350833a-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x08285bc1-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x787b4835-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0782edbf-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x08285bc1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x41937176 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x60a85739-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0cacb689-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x725ad586-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x2f43660d-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x013b2599-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x7a8e2896 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x67c7b603-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x2f43660d java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0782edbf java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x6b082c32 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x60a85739 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x4d41dcc7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x725ad586-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x2b5480af-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x4251adba java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x671a6cfd java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x2f43660d-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x4d41dcc7-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x21606365-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) - Thread LEAK? -, OpenFileDescriptor=653 (was 585) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=488 (was 531), ProcessCount=11 (was 11), AvailableMemoryMB=5969 (was 6088) 2024-12-02T14:20:56,478 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[0] Thread=377, OpenFileDescriptor=653, MaxFileDescriptor=1048576, SystemLoadAverage=488, ProcessCount=11, AvailableMemoryMB=5967 2024-12-02T14:20:56,480 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:20:56,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:20:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-02T14:20:56,483 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:56,485 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7ad0bd9b to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1f33fbaa 2024-12-02T14:20:56,488 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x7ad0bd9b {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7ad0bd9b to 127.0.0.1:53873 2024-12-02T14:20:56,488 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:56,497 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:56,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-02T14:20:56,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=94 2024-12-02T14:20:56,651 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=94}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-02T14:20:56,685 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=94}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:56,688 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-12-02T14:20:56,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-12-02T14:20:56,690 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:56,693 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-12-02T14:20:56,693 INFO [PEWorker-2 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-02T14:20:56,693 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 194 msec 2024-12-02T14:20:56,695 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 213 msec 2024-12-02T14:20:56,701 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:56,702 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x38aa8a1d to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@44731fbc 2024-12-02T14:20:56,705 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:56,705 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:56,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-12-02T14:20:56,800 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:20:56,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-02T14:20:56,808 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3992): Client=jenkins//172.17.0.2 disable replication peer, id=1 2024-12-02T14:20:56,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure 2024-12-02T14:20:56,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-02T14:20:56,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:56,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-02T14:20:56,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=96 2024-12-02T14:20:56,971 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=DISABLE_PEER 2024-12-02T14:20:56,973 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:20:56,973 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:20:56,973 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:56,974 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:56,978 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x64be65c6 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6204a30e 2024-12-02T14:20:56,979 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:56,997 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:56,997 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:20:56,997 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-02T14:20:56,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-12-02T14:20:57,000 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for DISABLE on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:57,002 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-12-02T14:20:57,002 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 184 msec 2024-12-02T14:20:57,002 INFO [PEWorker-3 {}] replication.DisablePeerProcedure(67): Successfully disabled peer 1 2024-12-02T14:20:57,003 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure in 194 msec 2024-12-02T14:20:57,012 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:57,013 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x564f483c to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3e318599 2024-12-02T14:20:57,015 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:57,016 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:57,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-12-02T14:20:57,129 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: DISABLE_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:20:57,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-02T14:20:57,131 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:20:57,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:57,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-02T14:20:57,136 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:20:57,137 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-02T14:20:57,139 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:20:57,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:20:57,143 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:20:57,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-02T14:20:57,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:20:57,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:20:57,145 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:57,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-02T14:20:57,299 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=98 2024-12-02T14:20:57,299 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-02T14:20:57,299 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-02T14:20:57,299 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Replication stream was removed by a user 2024-12-02T14:20:57,299 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:57,299 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:57,301 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5a51960b to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@60467e95 2024-12-02T14:20:57,301 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:57,302 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSourceManager(562): Done with the queue 1-a3a61c9ba14f,46013,1733149239111 2024-12-02T14:20:57,304 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-02T14:20:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-12-02T14:20:57,306 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:57,308 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-12-02T14:20:57,308 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-02T14:20:57,308 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 161 msec 2024-12-02T14:20:57,308 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-02T14:20:57,314 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-02T14:20:57,316 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 183 msec 2024-12-02T14:20:57,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-12-02T14:20:57,449 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:20:57,451 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:20:57,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:57,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T14:20:57,454 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:57,455 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=99, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:57,457 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=99, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-02T14:20:57,457 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:57,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-12-02T14:20:57,559 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-02T14:20:57,560 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:20:57,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=100, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:57,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100 2024-12-02T14:20:57,562 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:57,564 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=100, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:57,565 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=100, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-02T14:20:57,565 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:57,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=100 2024-12-02T14:20:57,669 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:20:57,684 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[0] Thread=389 (was 377) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x564f483c-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x64be65c6-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x38aa8a1d-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x38aa8a1d java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x564f483c java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x5a51960b java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x38aa8a1d-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x5a51960b-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x64be65c6-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x64be65c6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x5a51960b-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x564f483c-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) - Thread LEAK? -, OpenFileDescriptor=669 (was 653) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=488 (was 488), ProcessCount=11 (was 11), AvailableMemoryMB=5942 (was 5967) 2024-12-02T14:20:57,694 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[0] Thread=389, OpenFileDescriptor=669, MaxFileDescriptor=1048576, SystemLoadAverage=488, ProcessCount=11, AvailableMemoryMB=5941 2024-12-02T14:20:57,695 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=dummy_peer 2024-12-02T14:20:57,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:57,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T14:20:57,697 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer dummy_peer, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:57,698 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=101, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:57,698 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=101, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-02T14:20:57,698 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:57,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-12-02T14:20:57,810 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: dummy_peer failed with dummy_peer 2024-12-02T14:20:57,811 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:20:57,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=102, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:57,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=102 2024-12-02T14:20:57,815 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:57,816 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=102, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:57,819 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=102, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec 2024-12-02T14:20:57,819 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:57,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=102 2024-12-02T14:20:57,919 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-02T14:20:57,920 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:20:57,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:57,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T14:20:57,925 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:57,927 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=103, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:57,929 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=103, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=7 msec 2024-12-02T14:20:57,929 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:58,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-12-02T14:20:58,030 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:20:58,055 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[0] Thread=389 (was 389), OpenFileDescriptor=669 (was 669), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=488 (was 488), ProcessCount=11 (was 11), AvailableMemoryMB=5938 (was 5941) 2024-12-02T14:20:58,067 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerConfig[0] Thread=389, OpenFileDescriptor=669, MaxFileDescriptor=1048576, SystemLoadAverage=488, ProcessCount=11, AvailableMemoryMB=5937 2024-12-02T14:20:58,069 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:20:58,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=104, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:20:58,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-12-02T14:20:58,072 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:58,074 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x49ef7456 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5a7a416 2024-12-02T14:20:58,076 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x49ef7456 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x49ef7456 to 127.0.0.1:53873 2024-12-02T14:20:58,077 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:58,087 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=105, ppid=104, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:58,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-12-02T14:20:58,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=105 2024-12-02T14:20:58,239 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=105}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-02T14:20:58,264 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=105}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:58,266 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=105}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=105 2024-12-02T14:20:58,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=105 2024-12-02T14:20:58,269 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:58,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=105, resume processing ppid=104 2024-12-02T14:20:58,271 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, ppid=104, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 182 msec 2024-12-02T14:20:58,271 INFO [PEWorker-5 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-02T14:20:58,272 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 202 msec 2024-12-02T14:20:58,282 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:58,283 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0f530a61 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5bb410f6 2024-12-02T14:20:58,285 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:58,285 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:58,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=104 2024-12-02T14:20:58,390 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:20:58,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-02T14:20:58,394 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:20:58,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=106, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:58,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-02T14:20:58,401 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:20:58,402 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-02T14:20:58,403 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:20:58,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:20:58,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:20:58,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-02T14:20:58,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:20:58,408 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:20:58,410 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:58,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-02T14:20:58,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=107 2024-12-02T14:20:58,563 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-02T14:20:58,563 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-02T14:20:58,563 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Replication stream was removed by a user 2024-12-02T14:20:58,564 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:58,564 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:58,565 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1defbe60 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@53bd3539 2024-12-02T14:20:58,566 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:58,566 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.ReplicationSourceManager(562): Done with the queue 1-a3a61c9ba14f,46013,1733149239111 2024-12-02T14:20:58,569 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=107 2024-12-02T14:20:58,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=107 2024-12-02T14:20:58,572 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:58,573 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-12-02T14:20:58,573 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-02T14:20:58,573 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 163 msec 2024-12-02T14:20:58,573 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-02T14:20:58,579 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-02T14:20:58,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 185 msec 2024-12-02T14:20:58,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-12-02T14:20:58,719 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:20:58,719 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:20:58,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=108, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:58,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-12-02T14:20:58,723 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:58,724 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=108, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:58,726 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=108, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-02T14:20:58,726 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:58,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=108 2024-12-02T14:20:58,831 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-02T14:20:58,832 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:20:58,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:58,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T14:20:58,838 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:58,840 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=109, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:58,842 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=109, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=9 msec 2024-12-02T14:20:58,842 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:58,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-12-02T14:20:58,950 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:20:58,973 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerConfig[0] Thread=395 (was 389) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0f530a61 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x1defbe60 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x1defbe60-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0f530a61-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x1defbe60-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0f530a61-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) - Thread LEAK? -, OpenFileDescriptor=677 (was 669) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=488 (was 488), ProcessCount=11 (was 11), AvailableMemoryMB=5920 (was 5937) 2024-12-02T14:20:58,983 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[0] Thread=395, OpenFileDescriptor=677, MaxFileDescriptor=1048576, SystemLoadAverage=488, ProcessCount=11, AvailableMemoryMB=5919 2024-12-02T14:20:58,989 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:20:58,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:58,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] ipc.CallRunner(138): callId: 306 service: MasterService methodName: AddReplicationPeer size: 90 connection: 172.17.0.2:51456 deadline: 1733149318989, exception=java.io.IOException: Replication peer modification disabled 2024-12-02T14:20:59,013 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to master failed, tries = 1, maxAttempts = 2, timeout = 120000 ms, time elapsed = 22 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:465) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:20:59,118 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:20:59,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:59,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] ipc.CallRunner(138): callId: 307 service: MasterService methodName: AddReplicationPeer size: 90 connection: 172.17.0.2:51456 deadline: 1733149319118, exception=java.io.IOException: Replication peer modification disabled 2024-12-02T14:20:59,120 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to master failed, tries = 2, maxAttempts = 2, timeout = 120000 ms, time elapsed = 130 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:465) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:20:59,122 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Failed after attempts=2, exceptions: 2024-12-02T14:20:59.014Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:59.121Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:20:59,127 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:20:59,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=110, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:59,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-02T14:20:59,130 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:59,131 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=110, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:59,132 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=110, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-02T14:20:59,132 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:59,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=110 2024-12-02T14:20:59,239 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-02T14:20:59,240 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:20:59,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:59,244 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:20:59,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T14:20:59,247 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=111, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:20:59,249 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=111, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=8 msec 2024-12-02T14:20:59,249 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:20:59,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-12-02T14:20:59,360 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:20:59,384 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[0] Thread=395 (was 395), OpenFileDescriptor=677 (was 677), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=488 (was 488), ProcessCount=11 (was 11), AvailableMemoryMB=5913 (was 5919) 2024-12-02T14:20:59,398 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[0] Thread=395, OpenFileDescriptor=677, MaxFileDescriptor=1048576, SystemLoadAverage=488, ProcessCount=11, AvailableMemoryMB=5912 2024-12-02T14:20:59,399 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:20:59,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=112, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:20:59,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-02T14:20:59,403 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:59,404 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5e0a00eb to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@730e80f8 2024-12-02T14:20:59,407 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x5e0a00eb {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5e0a00eb to 127.0.0.1:53873 2024-12-02T14:20:59,407 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:20:59,419 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-02T14:20:59,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=113 2024-12-02T14:20:59,574 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=113}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-02T14:20:59,607 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=113}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:20:59,610 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=113}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=113 2024-12-02T14:20:59,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=113 2024-12-02T14:20:59,612 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:59,615 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-12-02T14:20:59,615 INFO [PEWorker-4 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-02T14:20:59,615 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 193 msec 2024-12-02T14:20:59,616 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 216 msec 2024-12-02T14:20:59,626 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:59,627 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x131a7c93 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@33904522 2024-12-02T14:20:59,630 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:20:59,630 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:20:59,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=112 2024-12-02T14:20:59,719 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:20:59,721 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:20:59,723 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:20:59,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=114, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:20:59,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-02T14:20:59,731 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:20:59,731 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-02T14:20:59,732 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:20:59,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:20:59,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:20:59,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-02T14:20:59,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:20:59,734 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:20:59,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:20:59,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-02T14:20:59,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=115 2024-12-02T14:20:59,889 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-02T14:20:59,889 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-02T14:20:59,890 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Replication stream was removed by a user 2024-12-02T14:20:59,890 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:20:59,890 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:20:59,893 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3fa833e6 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@261eeb46 2024-12-02T14:20:59,894 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:20:59,894 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.ReplicationSourceManager(562): Done with the queue 1-a3a61c9ba14f,46013,1733149239111 2024-12-02T14:20:59,898 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=115 2024-12-02T14:20:59,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=115 2024-12-02T14:20:59,901 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:20:59,904 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-12-02T14:20:59,904 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-02T14:20:59,905 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-02T14:20:59,905 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 166 msec 2024-12-02T14:20:59,912 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-02T14:20:59,915 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 188 msec 2024-12-02T14:21:00,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=114 2024-12-02T14:21:00,049 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:21:00,049 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:21:00,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:00,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-02T14:21:00,052 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:00,053 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=116, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:00,054 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=116, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-02T14:21:00,054 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:00,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-12-02T14:21:00,158 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:21:00,173 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[0] Thread=401 (was 395) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x131a7c93-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x3fa833e6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x131a7c93 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x131a7c93-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x3fa833e6-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x3fa833e6-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) - Thread LEAK? -, OpenFileDescriptor=685 (was 677) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=488 (was 488), ProcessCount=11 (was 11), AvailableMemoryMB=5895 (was 5912) 2024-12-02T14:21:00,185 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerBandwidth[0] Thread=401, OpenFileDescriptor=685, MaxFileDescriptor=1048576, SystemLoadAverage=488, ProcessCount=11, AvailableMemoryMB=5894 2024-12-02T14:21:00,186 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:21:00,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:21:00,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T14:21:00,190 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:00,191 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x51b36d6c to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@c6ece7e 2024-12-02T14:21:00,195 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x51b36d6c {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x51b36d6c to 127.0.0.1:53873 2024-12-02T14:21:00,195 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:00,206 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:00,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T14:21:00,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=118 2024-12-02T14:21:00,359 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=118}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-02T14:21:00,394 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=118}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:00,396 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-12-02T14:21:00,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-12-02T14:21:00,398 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:00,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-12-02T14:21:00,400 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 192 msec 2024-12-02T14:21:00,400 INFO [PEWorker-2 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-02T14:21:00,401 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 214 msec 2024-12-02T14:21:00,410 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:00,411 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3d431100 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3e7234ce 2024-12-02T14:21:00,414 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:00,414 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:00,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-12-02T14:21:00,509 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:21:00,510 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:00,513 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-12-02T14:21:00,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:21:00,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-02T14:21:00,518 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:00,520 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6f0df7bb to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@273de7ca 2024-12-02T14:21:00,523 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x6f0df7bb {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6f0df7bb to 127.0.0.1:53873 2024-12-02T14:21:00,523 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:00,526 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:00,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-02T14:21:00,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=120 2024-12-02T14:21:00,680 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:21:00,684 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-02T14:21:00,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-12-02T14:21:00,690 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:00,694 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-12-02T14:21:00,694 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 164 msec 2024-12-02T14:21:00,694 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-12-02T14:21:00,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 181 msec 2024-12-02T14:21:00,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-12-02T14:21:00,838 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:21:00,839 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:00,840 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:21:00,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:00,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T14:21:00,844 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:21:00,845 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-02T14:21:00,845 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:21:00,848 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:21:00,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:21:00,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-02T14:21:00,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:21:00,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:21:00,850 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:00,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T14:21:01,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=122 2024-12-02T14:21:01,003 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-02T14:21:01,004 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-02T14:21:01,004 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Replication stream was removed by a user 2024-12-02T14:21:01,004 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:01,004 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:01,006 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x105e63bb to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3ca33a6c 2024-12-02T14:21:01,007 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:01,008 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.ReplicationSourceManager(562): Done with the queue 1-a3a61c9ba14f,46013,1733149239111 2024-12-02T14:21:01,011 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-02T14:21:01,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-12-02T14:21:01,015 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:01,019 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-12-02T14:21:01,019 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 165 msec 2024-12-02T14:21:01,019 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-02T14:21:01,019 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-02T14:21:01,029 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-02T14:21:01,032 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 189 msec 2024-12-02T14:21:01,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-12-02T14:21:01,159 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:21:01,159 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:21:01,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:01,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T14:21:01,161 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:01,162 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=123, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:01,164 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=123, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-02T14:21:01,164 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:01,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-12-02T14:21:01,269 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-02T14:21:01,270 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:21:01,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=124, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-12-02T14:21:01,272 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:01,274 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=124, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:01,275 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=124, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-02T14:21:01,275 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:01,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=124 2024-12-02T14:21:01,379 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:21:01,394 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerBandwidth[0] Thread=406 (was 401) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x3d431100 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x105e63bb java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x105e63bb-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x3d431100-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x105e63bb-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x3d431100-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) - Thread LEAK? -, OpenFileDescriptor=693 (was 685) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=449 (was 488), ProcessCount=11 (was 11), AvailableMemoryMB=5872 (was 5894) 2024-12-02T14:21:01,404 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[0] Thread=406, OpenFileDescriptor=693, MaxFileDescriptor=1048576, SystemLoadAverage=449, ProcessCount=11, AvailableMemoryMB=5871 2024-12-02T14:21:01,405 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:21:01,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:21:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-02T14:21:01,415 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:01,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-02T14:21:01,562 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T14:21:01,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=126 2024-12-02T14:21:01,569 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=126}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-02T14:21:01,598 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=126}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:01,601 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-12-02T14:21:01,601 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(613): peerId=1, queueId=1-a3a61c9ba14f,46013,1733149239111 (queues=1) is replicating from cluster=5762d0f7-da11-4ede-9d35-feade635cfff to cluster=5762d0f7-da11-4ede-9d35-feade635cfff 2024-12-02T14:21:01,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-12-02T14:21:01,603 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(382): peerId=1, starting shipping worker for walGroupId=a3a61c9ba14f%2C46013%2C1733149239111 2024-12-02T14:21:01,603 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:01,605 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-12-02T14:21:01,605 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 188 msec 2024-12-02T14:21:01,605 INFO [PEWorker-1 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-02T14:21:01,607 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 200 msec 2024-12-02T14:21:01,607 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=1-a3a61c9ba14f,46013,1733149239111, ReplicationSourceWALReaderThread : 1 inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-12-02T14:21:01,609 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111.replicationSource.shippera3a61c9ba14f%2C46013%2C1733149239111,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSourceShipper(98): Running ReplicationSourceShipper Thread for wal group: a3a61c9ba14f%2C46013%2C1733149239111 2024-12-02T14:21:01,612 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111.replicationSource.wal-reader.a3a61c9ba14f%2C46013%2C1733149239111,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/WALs/a3a61c9ba14f,46013,1733149239111/a3a61c9ba14f%2C46013%2C1733149239111.1733149241428, startPosition=0, beingWritten=true 2024-12-02T14:21:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-12-02T14:21:01,728 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:21:01,729 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=2, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:21:01,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:21:01,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T14:21:01,732 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] ... 12 more 2024-12-02T14:21:01,735 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=127, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:01,737 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=127, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=6 msec 2024-12-02T14:21:01,737 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:01,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-12-02T14:21:01,840 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 2 failed with Invalid cluster key: 2024-12-02T14:21:01,841 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:21:01,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=128, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:01,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-02T14:21:01,850 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:21:01,850 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-02T14:21:01,850 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:21:01,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:21:01,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:21:01,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-02T14:21:01,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:21:01,853 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:21:01,854 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=129, ppid=128, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:01,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-02T14:21:02,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=129 2024-12-02T14:21:02,007 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-02T14:21:02,007 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-02T14:21:02,007 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Replication stream was removed by a user 2024-12-02T14:21:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-02T14:21:02,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-02T14:21:02,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-02T14:21:03,007 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111.replicationSource.wal-reader.a3a61c9ba14f%2C46013%2C1733149239111,1-a3a61c9ba14f,46013,1733149239111 {}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:130) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:162) ~[classes/:?] 2024-12-02T14:21:03,007 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111.replicationSource.shippera3a61c9ba14f%2C46013%2C1733149239111,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSourceShipper(123): Interrupted while waiting for next replication entry batch java.lang.InterruptedException: null at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1681) ~[?:?] at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) ~[?:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.poll(ReplicationSourceWALReader.java:309) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceShipper.run(ReplicationSourceShipper.java:109) ~[classes/:?] 2024-12-02T14:21:03,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-02T14:21:04,007 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.ReplicationSource(739): peerId=1, ReplicationSourceWorker RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111.replicationSource.shippera3a61c9ba14f%2C46013%2C1733149239111,1-a3a61c9ba14f,46013,1733149239111 terminated 2024-12-02T14:21:04,008 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.ReplicationSourceManager(562): Done with the queue 1-a3a61c9ba14f,46013,1733149239111 2024-12-02T14:21:04,010 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=129}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=129 2024-12-02T14:21:04,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=129 2024-12-02T14:21:04,012 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:04,014 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=129, resume processing ppid=128 2024-12-02T14:21:04,014 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-02T14:21:04,014 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-02T14:21:04,014 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, ppid=128, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 2.1580 sec 2024-12-02T14:21:04,020 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-02T14:21:04,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 2.1790 sec 2024-12-02T14:21:05,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=128 2024-12-02T14:21:05,998 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:21:05,999 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:21:06,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=130, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:06,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-12-02T14:21:06,004 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:06,006 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=130, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:06,007 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=130, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=7 msec 2024-12-02T14:21:06,007 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:06,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=130 2024-12-02T14:21:06,109 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:21:06,126 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[0] Thread=407 (was 406) - Thread LEAK? -, OpenFileDescriptor=698 (was 693) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=445 (was 449), ProcessCount=11 (was 11), AvailableMemoryMB=5839 (was 5871) 2024-12-02T14:21:06,139 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[1] Thread=407, OpenFileDescriptor=698, MaxFileDescriptor=1048576, SystemLoadAverage=445, ProcessCount=11, AvailableMemoryMB=5839 2024-12-02T14:21:06,142 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:21:06,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:21:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T14:21:06,145 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:06,146 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6aac679d to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4046f03e 2024-12-02T14:21:06,149 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x6aac679d {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6aac679d to 127.0.0.1:53873 2024-12-02T14:21:06,150 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:06,160 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:06,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T14:21:06,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=132 2024-12-02T14:21:06,313 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=132}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-02T14:21:06,346 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=132}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:06,348 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-12-02T14:21:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-12-02T14:21:06,353 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:06,355 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-12-02T14:21:06,355 INFO [PEWorker-1 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-02T14:21:06,355 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 193 msec 2024-12-02T14:21:06,356 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 213 msec 2024-12-02T14:21:06,364 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:06,366 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0e6a481f to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4201016 2024-12-02T14:21:06,368 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:06,368 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:06,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-12-02T14:21:06,458 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:21:06,459 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-02T14:21:06,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:21:06,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T14:21:06,462 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:06,463 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1f80646b to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6bb1b351 2024-12-02T14:21:06,482 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x1f80646b {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1f80646b to 127.0.0.1:53873 2024-12-02T14:21:06,482 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:06,488 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:06,510 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T14:21:06,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T14:21:06,641 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=134 2024-12-02T14:21:06,641 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:21:06,642 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:21:06,643 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:21:06,643 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:06,643 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:06,644 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x386a7052 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@193857c 2024-12-02T14:21:06,645 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:06,667 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:06,667 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:21:06,668 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-02T14:21:06,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-12-02T14:21:06,670 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:06,674 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-12-02T14:21:06,674 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-02T14:21:06,674 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 183 msec 2024-12-02T14:21:06,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 215 msec 2024-12-02T14:21:06,687 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:06,688 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x78f03a1e to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@8c38d28 2024-12-02T14:21:06,691 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:06,692 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:06,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-12-02T14:21:06,780 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:21:06,781 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:06,782 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:06,783 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:06,784 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-02T14:21:06,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:21:06,787 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:06,789 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5c6a166a to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5523252e 2024-12-02T14:21:06,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T14:21:06,796 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x5c6a166a {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5c6a166a to 127.0.0.1:53873 2024-12-02T14:21:06,796 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:06,802 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:06,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T14:21:06,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=136 2024-12-02T14:21:06,955 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:21:06,956 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:21:06,956 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:21:06,956 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:06,956 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:06,958 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2979c9b9 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6663fe0f 2024-12-02T14:21:06,960 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:06,983 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:06,983 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:21:06,983 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-02T14:21:06,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-12-02T14:21:06,986 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:06,992 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-12-02T14:21:06,992 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-02T14:21:06,992 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 185 msec 2024-12-02T14:21:06,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 208 msec 2024-12-02T14:21:07,001 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:07,002 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1e6fead1 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@430823a6 2024-12-02T14:21:07,006 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:07,006 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:07,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-12-02T14:21:07,108 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:21:07,109 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:07,110 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:07,111 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:07,111 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:07,112 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-02T14:21:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:21:07,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T14:21:07,115 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:07,116 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x209bd146 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@33d69fba 2024-12-02T14:21:07,118 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x209bd146 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x209bd146 to 127.0.0.1:53873 2024-12-02T14:21:07,118 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:07,121 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:07,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T14:21:07,274 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=138 2024-12-02T14:21:07,274 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:21:07,275 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:21:07,275 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:21:07,275 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:07,275 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:07,277 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x43fccd5b to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@168ddad0 2024-12-02T14:21:07,277 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:07,293 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:07,293 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:21:07,293 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-12-02T14:21:07,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-12-02T14:21:07,296 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:07,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-12-02T14:21:07,298 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-02T14:21:07,298 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 175 msec 2024-12-02T14:21:07,299 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 186 msec 2024-12-02T14:21:07,314 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:07,315 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x624fb538 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@17b57956 2024-12-02T14:21:07,318 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:07,318 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:07,325 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-02T14:21:07,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-12-02T14:21:07,428 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:21:07,428 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:07,429 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:07,430 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:07,431 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-02T14:21:07,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:21:07,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-02T14:21:07,435 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:07,436 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x19277a1a to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2b8475fe 2024-12-02T14:21:07,443 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x19277a1a {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x19277a1a to 127.0.0.1:53873 2024-12-02T14:21:07,443 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:07,447 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:07,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-02T14:21:07,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=140 2024-12-02T14:21:07,601 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:21:07,604 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:21:07,604 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:21:07,604 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:07,604 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:07,606 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4a9ccae5 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@640f8ad3 2024-12-02T14:21:07,610 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:07,668 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:07,668 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:21:07,669 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-12-02T14:21:07,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=140 2024-12-02T14:21:07,672 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:07,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-12-02T14:21:07,674 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-02T14:21:07,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 225 msec 2024-12-02T14:21:07,676 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 243 msec 2024-12-02T14:21:07,690 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:07,692 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x76c0df9d to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2ed6373f 2024-12-02T14:21:07,702 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:07,702 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:07,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-12-02T14:21:07,749 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:21:07,749 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:07,750 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:07,751 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-02T14:21:07,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:21:07,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-02T14:21:07,754 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:07,755 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x35dcacac to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@30aab83d 2024-12-02T14:21:07,757 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x35dcacac {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x35dcacac to 127.0.0.1:53873 2024-12-02T14:21:07,758 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:07,761 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:07,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-02T14:21:07,913 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=142 2024-12-02T14:21:07,914 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:21:07,915 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:21:07,915 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:21:07,915 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:07,915 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:07,916 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x02b4e8d9 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@57fda27b 2024-12-02T14:21:07,916 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:07,937 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:07,937 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:21:07,937 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-12-02T14:21:07,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=142 2024-12-02T14:21:07,940 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:07,941 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-12-02T14:21:07,941 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 179 msec 2024-12-02T14:21:07,941 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-02T14:21:07,943 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 190 msec 2024-12-02T14:21:07,951 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:07,952 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0b1cd25a to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3bd88af 2024-12-02T14:21:07,954 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:07,954 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:08,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-12-02T14:21:08,068 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:21:08,069 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:08,070 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-02T14:21:08,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:21:08,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-02T14:21:08,073 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:08,074 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x768579f3 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@387a309d 2024-12-02T14:21:08,076 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x768579f3 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x768579f3 to 127.0.0.1:53873 2024-12-02T14:21:08,076 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:08,079 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:08,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-02T14:21:08,231 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=144 2024-12-02T14:21:08,232 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:21:08,233 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:21:08,233 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:21:08,233 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:08,233 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:08,235 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3a49dcf5 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@160a5987 2024-12-02T14:21:08,235 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:08,249 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:08,249 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:21:08,250 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-12-02T14:21:08,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=144 2024-12-02T14:21:08,252 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:08,254 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-12-02T14:21:08,254 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-02T14:21:08,254 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 173 msec 2024-12-02T14:21:08,255 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 184 msec 2024-12-02T14:21:08,262 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:08,263 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x354da2bd to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7efaf750 2024-12-02T14:21:08,265 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:08,265 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:08,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-12-02T14:21:08,388 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:21:08,388 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:08,389 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:21:08,389 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:21:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:08,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:08,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-02T14:21:08,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-02T14:21:08,393 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:21:08,394 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-02T14:21:08,394 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:21:08,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:21:08,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:21:08,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-02T14:21:08,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:21:08,396 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:21:08,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:08,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-02T14:21:08,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-02T14:21:08,550 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=147 2024-12-02T14:21:08,550 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-02T14:21:08,550 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-02T14:21:08,550 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Replication stream was removed by a user 2024-12-02T14:21:08,550 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:08,550 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:08,551 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4153c0ef to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@55f4cd0f 2024-12-02T14:21:08,552 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:08,552 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.ReplicationSourceManager(562): Done with the queue 1-a3a61c9ba14f,46013,1733149239111 2024-12-02T14:21:08,554 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-12-02T14:21:08,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-12-02T14:21:08,556 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:08,557 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-12-02T14:21:08,557 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 159 msec 2024-12-02T14:21:08,557 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-02T14:21:08,558 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-02T14:21:08,563 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-02T14:21:08,564 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 174 msec 2024-12-02T14:21:08,565 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:08,566 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=145, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:08,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=145, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=177 msec 2024-12-02T14:21:08,567 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:08,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-12-02T14:21:08,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-12-02T14:21:08,708 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-02T14:21:08,709 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:21:08,709 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:21:08,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:08,711 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:08,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-02T14:21:08,712 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=148, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:08,713 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=148, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-02T14:21:08,713 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:08,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-12-02T14:21:08,818 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:21:08,837 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[1] Thread=453 (was 407) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x624fb538 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x4a9ccae5-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x4153c0ef java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x2979c9b9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x78f03a1e java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0b1cd25a-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x624fb538-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x78f03a1e-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x43fccd5b-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x1e6fead1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x3a49dcf5-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x4153c0ef-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x1e6fead1-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x78f03a1e-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x4a9ccae5-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x3a49dcf5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x2979c9b9-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x76c0df9d-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x02b4e8d9-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0b1cd25a-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x3a49dcf5-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x76c0df9d java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x4a9ccae5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0b1cd25a java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0e6a481f-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x1e6fead1-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x386a7052-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0e6a481f-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x2979c9b9-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x354da2bd-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x624fb538-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x02b4e8d9-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x386a7052-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x43fccd5b java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x43fccd5b-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x354da2bd java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0e6a481f java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x386a7052 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x4153c0ef-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x02b4e8d9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x354da2bd-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x76c0df9d-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) - Thread LEAK? -, OpenFileDescriptor=755 (was 698) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=445 (was 445), ProcessCount=11 (was 11), AvailableMemoryMB=5751 (was 5839) 2024-12-02T14:21:08,849 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[1] Thread=453, OpenFileDescriptor=755, MaxFileDescriptor=1048576, SystemLoadAverage=445, ProcessCount=11, AvailableMemoryMB=5750 2024-12-02T14:21:08,851 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:21:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=149, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:21:08,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-02T14:21:08,854 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:08,855 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6b2f2f82 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@21c42553 2024-12-02T14:21:08,859 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x6b2f2f82 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6b2f2f82 to 127.0.0.1:53873 2024-12-02T14:21:08,859 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:08,869 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:08,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-02T14:21:09,022 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=150 2024-12-02T14:21:09,022 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=150}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-02T14:21:09,062 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=150}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:09,065 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=150}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=150 2024-12-02T14:21:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=150 2024-12-02T14:21:09,068 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:09,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-12-02T14:21:09,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 199 msec 2024-12-02T14:21:09,071 INFO [PEWorker-3 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-02T14:21:09,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 220 msec 2024-12-02T14:21:09,087 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:09,089 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7fa7ec82 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6153df7f 2024-12-02T14:21:09,097 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:09,097 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:09,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=149 2024-12-02T14:21:09,168 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:21:09,169 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-02T14:21:09,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=151, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:21:09,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-12-02T14:21:09,171 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:09,172 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5ef620c8 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@53c6af0a 2024-12-02T14:21:09,175 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x5ef620c8 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5ef620c8 to 127.0.0.1:53873 2024-12-02T14:21:09,175 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:09,178 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:09,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-12-02T14:21:09,330 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=152 2024-12-02T14:21:09,331 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:21:09,332 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:21:09,332 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:21:09,332 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:09,332 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:09,333 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2edbd17b to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1b7000d5 2024-12-02T14:21:09,334 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:09,355 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:09,355 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:21:09,355 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=152 2024-12-02T14:21:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=152 2024-12-02T14:21:09,357 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:09,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=152, resume processing ppid=151 2024-12-02T14:21:09,359 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, ppid=151, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 179 msec 2024-12-02T14:21:09,359 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-02T14:21:09,360 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 190 msec 2024-12-02T14:21:09,373 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:09,374 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x42b79c32 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1b5cd3ff 2024-12-02T14:21:09,377 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:09,377 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=151 2024-12-02T14:21:09,489 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:21:09,489 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-02T14:21:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=153, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:21:09,493 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:09,494 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x413c268c to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6ba78a42 2024-12-02T14:21:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-12-02T14:21:09,497 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x413c268c {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x413c268c to 127.0.0.1:53873 2024-12-02T14:21:09,497 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:09,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:09,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-12-02T14:21:09,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=154 2024-12-02T14:21:09,655 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:21:09,656 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:21:09,657 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:21:09,657 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:09,657 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:09,658 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5ae7b7f4 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4d838d3d 2024-12-02T14:21:09,658 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:09,674 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:09,674 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:21:09,675 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=154 2024-12-02T14:21:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=154 2024-12-02T14:21:09,677 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:09,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=154, resume processing ppid=153 2024-12-02T14:21:09,680 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-02T14:21:09,680 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, ppid=153, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 175 msec 2024-12-02T14:21:09,681 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 190 msec 2024-12-02T14:21:09,689 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:09,690 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x70424add to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1f70b598 2024-12-02T14:21:09,692 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:09,693 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:09,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=153 2024-12-02T14:21:09,808 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:21:09,809 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns1:testNamespacesAndTableCfsConfigConflict1=null},bandwidth=0,serial=false 2024-12-02T14:21:09,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=155, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:21:09,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=155 2024-12-02T14:21:09,812 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:09,814 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x17e4084f to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7f6f5e56 2024-12-02T14:21:09,816 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x17e4084f {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x17e4084f to 127.0.0.1:53873 2024-12-02T14:21:09,816 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:09,817 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:09,818 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=155, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:09,819 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=155, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=9 msec 2024-12-02T14:21:09,819 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:09,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=155 2024-12-02T14:21:09,919 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config 2024-12-02T14:21:09,920 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-02T14:21:09,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:21:09,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-02T14:21:09,923 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:09,923 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6d03188a to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@63fe26c7 2024-12-02T14:21:09,930 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x6d03188a {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6d03188a to 127.0.0.1:53873 2024-12-02T14:21:09,930 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:09,934 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:10,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-02T14:21:10,087 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=157 2024-12-02T14:21:10,087 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=157}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:21:10,088 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=157}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:21:10,088 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=157}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:21:10,089 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:10,089 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:10,090 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x65ef7944 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@21779106 2024-12-02T14:21:10,091 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:10,112 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=157}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:10,112 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=157}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:21:10,112 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-12-02T14:21:10,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-12-02T14:21:10,115 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:10,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-12-02T14:21:10,117 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-02T14:21:10,117 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 181 msec 2024-12-02T14:21:10,119 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 197 msec 2024-12-02T14:21:10,130 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:10,132 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x45c46794 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@79cad92a 2024-12-02T14:21:10,134 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:10,135 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:10,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-12-02T14:21:10,239 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:21:10,239 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-02T14:21:10,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:21:10,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T14:21:10,242 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:10,243 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2fce7ef7 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@355da268 2024-12-02T14:21:10,246 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x2fce7ef7 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2fce7ef7 to 127.0.0.1:53873 2024-12-02T14:21:10,246 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:10,247 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:10,248 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=158, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:593) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:503) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:10,249 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=158, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=9 msec 2024-12-02T14:21:10,249 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:10,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-12-02T14:21:10,349 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config 2024-12-02T14:21:10,349 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:21:10,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=159, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:10,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=159 2024-12-02T14:21:10,352 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:21:10,353 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-02T14:21:10,353 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:21:10,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:21:10,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:21:10,355 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-02T14:21:10,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:21:10,356 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:21:10,357 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=160, ppid=159, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:10,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=159 2024-12-02T14:21:10,509 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=160 2024-12-02T14:21:10,510 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=160}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-02T14:21:10,510 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=160}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-02T14:21:10,510 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=160}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Replication stream was removed by a user 2024-12-02T14:21:10,510 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:10,510 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:10,512 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3049ba50 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@14f1cb11 2024-12-02T14:21:10,512 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:10,513 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=160}] regionserver.ReplicationSourceManager(562): Done with the queue 1-a3a61c9ba14f,46013,1733149239111 2024-12-02T14:21:10,515 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=160}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=160 2024-12-02T14:21:10,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=160 2024-12-02T14:21:10,517 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:10,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=160, resume processing ppid=159 2024-12-02T14:21:10,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, ppid=159, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 160 msec 2024-12-02T14:21:10,518 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-02T14:21:10,518 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-02T14:21:10,524 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-02T14:21:10,525 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 175 msec 2024-12-02T14:21:10,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=159 2024-12-02T14:21:10,668 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:21:10,669 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:21:10,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=161, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:10,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=161 2024-12-02T14:21:10,671 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:10,672 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=161, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:10,673 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=161, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-02T14:21:10,673 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:10,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=161 2024-12-02T14:21:10,779 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-02T14:21:10,779 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:21:10,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:10,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-02T14:21:10,781 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:10,782 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=162, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:10,783 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=162, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-02T14:21:10,783 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:10,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-12-02T14:21:10,889 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:21:10,910 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[1] Thread=477 (was 453) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x42b79c32-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x42b79c32-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x45c46794 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x70424add java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x2edbd17b-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x5ae7b7f4-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x5ae7b7f4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x3049ba50 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x5ae7b7f4-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x45c46794-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x65ef7944 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x7fa7ec82 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x70424add-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x65ef7944-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x65ef7944-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x42b79c32 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x45c46794-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x3049ba50-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x70424add-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x7fa7ec82-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x2edbd17b-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x3049ba50-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x2edbd17b java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x7fa7ec82-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) - Thread LEAK? -, OpenFileDescriptor=787 (was 755) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=445 (was 445), ProcessCount=11 (was 11), AvailableMemoryMB=5739 (was 5750) 2024-12-02T14:21:10,924 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAddRemovePeer[1] Thread=477, OpenFileDescriptor=787, MaxFileDescriptor=1048576, SystemLoadAverage=433, ProcessCount=11, AvailableMemoryMB=5739 2024-12-02T14:21:10,925 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:21:10,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=163, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:21:10,929 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:10,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=163 2024-12-02T14:21:10,930 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x01fe48d3 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@75c9c8cc 2024-12-02T14:21:10,942 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x01fe48d3 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x01fe48d3 to 127.0.0.1:53873 2024-12-02T14:21:10,942 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:10,951 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:11,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=163 2024-12-02T14:21:11,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=164 2024-12-02T14:21:11,104 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=164}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-02T14:21:11,143 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=164}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:11,146 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=164}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=164 2024-12-02T14:21:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=164 2024-12-02T14:21:11,148 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:11,151 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-12-02T14:21:11,151 INFO [PEWorker-5 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-02T14:21:11,151 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 197 msec 2024-12-02T14:21:11,152 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 226 msec 2024-12-02T14:21:11,165 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:11,166 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x64d2e993 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@417627a5 2024-12-02T14:21:11,168 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:11,169 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:11,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=163 2024-12-02T14:21:11,249 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:21:11,249 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:21:11,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=165, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:21:11,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=165 2024-12-02T14:21:11,252 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:11,252 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0a06994e to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5565a751 2024-12-02T14:21:11,254 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x0a06994e {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0a06994e to 127.0.0.1:53873 2024-12-02T14:21:11,254 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:11,255 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:177) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:11,256 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=165, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:177) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:11,257 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=165, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=7 msec 2024-12-02T14:21:11,257 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:11,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=165 2024-12-02T14:21:11,359 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Replication peer 1 already exists 2024-12-02T14:21:11,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-02T14:21:11,360 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:21:11,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:11,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T14:21:11,362 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:11,363 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=166, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:11,364 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=166, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-02T14:21:11,364 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:11,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-12-02T14:21:11,468 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:21:11,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-02T14:21:11,470 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=2, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:21:11,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=167, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:21:11,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=167 2024-12-02T14:21:11,472 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test2' 2024-12-02T14:21:11,473 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7b33abe0 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@163d0e8b 2024-12-02T14:21:11,476 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x7b33abe0 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7b33abe0 to 127.0.0.1:53873 2024-12-02T14:21:11,476 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test2 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:11,483 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=168, ppid=167, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:11,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=167 2024-12-02T14:21:11,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=168 2024-12-02T14:21:11,635 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=168}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=2, type=ADD_PEER 2024-12-02T14:21:11,673 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=168}] regionserver.ReplicationSource(231): queueId=2-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 2, currentBandwidth=0 2024-12-02T14:21:11,675 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=168}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=168 2024-12-02T14:21:11,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=168 2024-12-02T14:21:11,677 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 2 for ADD on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:11,679 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=168, resume processing ppid=167 2024-12-02T14:21:11,679 INFO [PEWorker-3 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 2, config clusterKey=hbase+zk://127.0.0.1:53873/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-02T14:21:11,679 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, ppid=167, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 194 msec 2024-12-02T14:21:11,680 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 209 msec 2024-12-02T14:21:11,689 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,2-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test2' 2024-12-02T14:21:11,690 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,2-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x27ad4815 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3e8aa175 2024-12-02T14:21:11,694 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,2-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:11,694 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,2-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=2, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=167 2024-12-02T14:21:11,788 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 2 completed 2024-12-02T14:21:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-02T14:21:11,789 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:21:11,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=169, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:11,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=169 2024-12-02T14:21:11,793 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:21:11,793 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-02T14:21:11,793 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:21:11,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:21:11,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:21:11,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-02T14:21:11,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:21:11,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:21:11,796 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:11,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=169 2024-12-02T14:21:11,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=170 2024-12-02T14:21:11,948 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-02T14:21:11,948 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-02T14:21:11,948 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Replication stream was removed by a user 2024-12-02T14:21:11,948 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:11,948 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:11,950 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x34dda0cb to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7c7b17a5 2024-12-02T14:21:11,950 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:11,950 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.ReplicationSourceManager(562): Done with the queue 1-a3a61c9ba14f,46013,1733149239111 2024-12-02T14:21:11,953 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=170 2024-12-02T14:21:11,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=170 2024-12-02T14:21:11,955 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:11,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=170, resume processing ppid=169 2024-12-02T14:21:11,956 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-02T14:21:11,956 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, ppid=169, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 159 msec 2024-12-02T14:21:11,956 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-02T14:21:11,961 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-02T14:21:11,963 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 172 msec 2024-12-02T14:21:12,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=169 2024-12-02T14:21:12,109 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:21:12,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-02T14:21:12,110 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:21:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=171, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:12,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=171 2024-12-02T14:21:12,113 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/sync-rep-state 2024-12-02T14:21:12,114 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/peer-state 2024-12-02T14:21:12,114 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/new-sync-rep-state 2024-12-02T14:21:12,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/new-sync-rep-state 2024-12-02T14:21:12,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/2 2024-12-02T14:21:12,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/peer-state 2024-12-02T14:21:12,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/sync-rep-state 2024-12-02T14:21:12,116 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2 2024-12-02T14:21:12,117 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=172, ppid=171, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:12,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=171 2024-12-02T14:21:12,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=172 2024-12-02T14:21:12,270 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=2, type=REMOVE_PEER 2024-12-02T14:21:12,270 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 2: 0 2024-12-02T14:21:12,270 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.ReplicationSource(687): peerId=2, Closing source 2-a3a61c9ba14f,46013,1733149239111 because: Replication stream was removed by a user 2024-12-02T14:21:12,270 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,2-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=2, Interrupted while sleeping between retries 2024-12-02T14:21:12,270 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,2-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test2' 2024-12-02T14:21:12,271 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,2-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x07e5e4d6 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@a35d501 2024-12-02T14:21:12,272 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,2-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:12,272 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.ReplicationSourceManager(562): Done with the queue 2-a3a61c9ba14f,46013,1733149239111 2024-12-02T14:21:12,274 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=172 2024-12-02T14:21:12,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=172 2024-12-02T14:21:12,277 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 2 for REMOVE on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:12,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=172, resume processing ppid=171 2024-12-02T14:21:12,280 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 2, move on 2024-12-02T14:21:12,280 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, ppid=171, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 160 msec 2024-12-02T14:21:12,280 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 2, move on 2024-12-02T14:21:12,286 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(120): Successfully removed peer 2 2024-12-02T14:21:12,287 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 176 msec 2024-12-02T14:21:12,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=171 2024-12-02T14:21:12,428 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 2 completed 2024-12-02T14:21:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-02T14:21:12,430 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:21:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=173, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=173 2024-12-02T14:21:12,431 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:12,432 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=173, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:12,433 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=173, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-02T14:21:12,433 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=173 2024-12-02T14:21:12,539 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-02T14:21:12,539 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:21:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=174, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-02T14:21:12,541 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:12,542 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=174, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:12,544 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=174, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-02T14:21:12,544 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=174 2024-12-02T14:21:12,648 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:21:12,666 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAddRemovePeer[1] Thread=489 (was 477) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x64d2e993 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x07e5e4d6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x64d2e993-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x07e5e4d6-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x07e5e4d6-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x34dda0cb-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x34dda0cb java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x34dda0cb-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x64d2e993-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x27ad4815-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x27ad4815-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x27ad4815 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=803 (was 787) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=433 (was 433), ProcessCount=11 (was 11), AvailableMemoryMB=5736 (was 5739) 2024-12-02T14:21:12,678 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[1] Thread=489, OpenFileDescriptor=803, MaxFileDescriptor=1048576, SystemLoadAverage=433, ProcessCount=11, AvailableMemoryMB=5735 2024-12-02T14:21:12,679 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=whatever,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:21:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=175, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:21:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=175 2024-12-02T14:21:12,681 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:470) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:467) ~[classes/:?] ... 11 more 2024-12-02T14:21:12,682 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=175, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:470) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:467) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:12,683 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=175, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=3 msec 2024-12-02T14:21:12,683 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:12,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=175 2024-12-02T14:21:12,789 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Can not instantiate configured replication endpoint class=whatever 2024-12-02T14:21:12,790 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:21:12,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=176, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:12,791 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:12,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-12-02T14:21:12,792 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=176, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:12,793 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=176, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-02T14:21:12,793 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:12,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=176 2024-12-02T14:21:12,899 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-02T14:21:12,899 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:21:12,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=177, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:12,901 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:12,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=177 2024-12-02T14:21:12,902 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=177, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:12,903 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=177, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-02T14:21:12,903 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:13,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=177 2024-12-02T14:21:13,009 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:21:13,027 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[1] Thread=489 (was 489), OpenFileDescriptor=803 (was 803), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=433 (was 433), ProcessCount=11 (was 11), AvailableMemoryMB=5735 (was 5735) 2024-12-02T14:21:13,039 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[1] Thread=489, OpenFileDescriptor=803, MaxFileDescriptor=1048576, SystemLoadAverage=433, ProcessCount=11, AvailableMemoryMB=5734 2024-12-02T14:21:13,040 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=whatever,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:21:13,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=178, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:21:13,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-12-02T14:21:13,042 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] ... 12 more 2024-12-02T14:21:13,043 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=178, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:13,044 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=178, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=3 msec 2024-12-02T14:21:13,044 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:13,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=178 2024-12-02T14:21:13,149 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Invalid cluster key: whatever 2024-12-02T14:21:13,149 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:21:13,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=179, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:13,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=179 2024-12-02T14:21:13,152 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:13,153 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=179, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:13,154 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=179, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-02T14:21:13,154 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:13,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=179 2024-12-02T14:21:13,259 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-02T14:21:13,259 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:21:13,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=180, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:13,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-12-02T14:21:13,261 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:13,262 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=180, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:13,263 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=180, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-02T14:21:13,263 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:13,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=180 2024-12-02T14:21:13,369 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:21:13,385 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[1] Thread=489 (was 489), OpenFileDescriptor=803 (was 803), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=433 (was 433), ProcessCount=11 (was 11), AvailableMemoryMB=5734 (was 5734) 2024-12-02T14:21:13,397 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[1] Thread=489, OpenFileDescriptor=803, MaxFileDescriptor=1048576, SystemLoadAverage=433, ProcessCount=11, AvailableMemoryMB=5734 2024-12-02T14:21:13,398 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:21:13,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=181, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:21:13,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=181 2024-12-02T14:21:13,400 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:13,401 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x34054d07 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@46013cf5 2024-12-02T14:21:13,403 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x34054d07 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x34054d07 to 127.0.0.1:53873 2024-12-02T14:21:13,403 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:13,411 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=182, ppid=181, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:13,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=181 2024-12-02T14:21:13,564 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=182 2024-12-02T14:21:13,564 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=182}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-02T14:21:13,601 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=182}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:13,604 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=182}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=182 2024-12-02T14:21:13,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=182 2024-12-02T14:21:13,606 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:13,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=182, resume processing ppid=181 2024-12-02T14:21:13,607 INFO [PEWorker-3 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-02T14:21:13,607 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=182, ppid=181, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 195 msec 2024-12-02T14:21:13,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=181, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 209 msec 2024-12-02T14:21:13,618 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:13,619 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x155b518d to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@20bf8623 2024-12-02T14:21:13,621 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:13,622 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:13,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=181 2024-12-02T14:21:13,718 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:21:13,718 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-02T14:21:13,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=183, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:21:13,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=183 2024-12-02T14:21:13,721 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:13,722 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x60888b35 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5d4d2fc6 2024-12-02T14:21:13,724 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x60888b35 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x60888b35 to 127.0.0.1:53873 2024-12-02T14:21:13,724 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:13,727 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=184, ppid=183, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:13,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=183 2024-12-02T14:21:13,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=184 2024-12-02T14:21:13,879 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:21:13,880 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:21:13,880 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:21:13,880 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:13,880 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:13,881 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7fae8571 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7d3c385b 2024-12-02T14:21:13,881 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:13,896 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:13,897 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:21:13,897 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=184 2024-12-02T14:21:13,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=184 2024-12-02T14:21:13,900 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:13,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=184, resume processing ppid=183 2024-12-02T14:21:13,902 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-02T14:21:13,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=184, ppid=183, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 173 msec 2024-12-02T14:21:13,903 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=183, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 184 msec 2024-12-02T14:21:13,912 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:13,912 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x01c33d68 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@656d2d88 2024-12-02T14:21:13,916 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:13,916 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:14,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=183 2024-12-02T14:21:14,038 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:21:14,039 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-12-02T14:21:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=185, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:21:14,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=185 2024-12-02T14:21:14,041 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:14,042 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x194d35cd to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@35e269f3 2024-12-02T14:21:14,044 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x194d35cd {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x194d35cd to 127.0.0.1:53873 2024-12-02T14:21:14,044 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:14,046 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=186, ppid=185, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:14,054 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T14:21:14,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=185 2024-12-02T14:21:14,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=186 2024-12-02T14:21:14,199 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:21:14,200 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:21:14,200 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:21:14,200 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:14,200 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:14,201 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x56c1a1dd to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@fff84e2 2024-12-02T14:21:14,202 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:14,222 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:14,222 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:21:14,222 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=186 2024-12-02T14:21:14,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=186 2024-12-02T14:21:14,225 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:14,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=186, resume processing ppid=185 2024-12-02T14:21:14,228 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-12-02T14:21:14,228 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=186, ppid=185, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 179 msec 2024-12-02T14:21:14,229 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=185, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 189 msec 2024-12-02T14:21:14,238 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:14,240 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x062b02fc to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2999fa9 2024-12-02T14:21:14,242 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:14,242 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:14,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=185 2024-12-02T14:21:14,358 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:21:14,359 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:14,360 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-02T14:21:14,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=187, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:21:14,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=187 2024-12-02T14:21:14,362 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:14,363 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7a1c9f21 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@13527911 2024-12-02T14:21:14,365 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x7a1c9f21 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7a1c9f21 to 127.0.0.1:53873 2024-12-02T14:21:14,365 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:14,368 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=188, ppid=187, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:14,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=187 2024-12-02T14:21:14,520 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=188 2024-12-02T14:21:14,520 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:21:14,521 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:21:14,521 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:21:14,522 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:14,522 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:14,523 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6582fddc to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@d899583 2024-12-02T14:21:14,524 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:14,545 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:14,545 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:21:14,545 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=188 2024-12-02T14:21:14,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=188 2024-12-02T14:21:14,548 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:14,549 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=188, resume processing ppid=187 2024-12-02T14:21:14,549 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=188, ppid=187, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 180 msec 2024-12-02T14:21:14,549 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-02T14:21:14,550 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=187, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 189 msec 2024-12-02T14:21:14,564 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:14,565 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1a50687d to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@11d3b568 2024-12-02T14:21:14,567 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:14,567 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:14,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=187 2024-12-02T14:21:14,678 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:21:14,678 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:14,679 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:21:14,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=189, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:14,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=189 2024-12-02T14:21:14,682 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:21:14,683 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-02T14:21:14,683 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:21:14,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:21:14,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:21:14,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-02T14:21:14,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:21:14,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:21:14,686 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=190, ppid=189, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:14,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=189 2024-12-02T14:21:14,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=190 2024-12-02T14:21:14,839 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-02T14:21:14,839 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-02T14:21:14,839 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Replication stream was removed by a user 2024-12-02T14:21:14,839 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:14,839 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:14,841 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2f273e43 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1a8a0358 2024-12-02T14:21:14,841 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:14,841 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSourceManager(562): Done with the queue 1-a3a61c9ba14f,46013,1733149239111 2024-12-02T14:21:14,843 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=190 2024-12-02T14:21:14,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=190 2024-12-02T14:21:14,845 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:14,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=190, resume processing ppid=189 2024-12-02T14:21:14,847 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-02T14:21:14,847 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=190, ppid=189, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 159 msec 2024-12-02T14:21:14,847 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-02T14:21:14,854 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-02T14:21:14,856 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=189, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 175 msec 2024-12-02T14:21:14,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=189 2024-12-02T14:21:14,998 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:21:14,998 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:21:14,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=191, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:15,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=191 2024-12-02T14:21:15,000 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:15,001 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=191, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:15,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=191, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-02T14:21:15,002 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:15,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=191 2024-12-02T14:21:15,109 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-02T14:21:15,110 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:21:15,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=192, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:15,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=192 2024-12-02T14:21:15,114 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:15,115 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=192, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:15,117 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=192, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec 2024-12-02T14:21:15,117 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=192 2024-12-02T14:21:15,218 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:21:15,246 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[1] Thread=513 (was 489) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x6582fddc-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x2f273e43 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x56c1a1dd java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x2f273e43-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x062b02fc-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x6582fddc java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x2f273e43-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x1a50687d-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x56c1a1dd-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x1a50687d-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x155b518d-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x7fae8571-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x7fae8571 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x155b518d-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x1a50687d java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x01c33d68 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x062b02fc java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x155b518d java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x01c33d68-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x6582fddc-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x062b02fc-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x7fae8571-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x01c33d68-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x56c1a1dd-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) - Thread LEAK? -, OpenFileDescriptor=833 (was 803) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=433 (was 433), ProcessCount=11 (was 11), AvailableMemoryMB=5719 (was 5734) 2024-12-02T14:21:15,248 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=513 is superior to 500 2024-12-02T14:21:15,265 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[1] Thread=513, OpenFileDescriptor=833, MaxFileDescriptor=1048576, SystemLoadAverage=433, ProcessCount=11, AvailableMemoryMB=5716 2024-12-02T14:21:15,265 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=513 is superior to 500 2024-12-02T14:21:15,266 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:21:15,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=193, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:21:15,268 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:15,269 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1b256f90 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4265bf8a 2024-12-02T14:21:15,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=193 2024-12-02T14:21:15,271 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x1b256f90 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1b256f90 to 127.0.0.1:53873 2024-12-02T14:21:15,272 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:15,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=194, ppid=193, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:15,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=193 2024-12-02T14:21:15,438 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=194 2024-12-02T14:21:15,439 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=194}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-02T14:21:15,479 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=194}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:15,482 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=194}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=194 2024-12-02T14:21:15,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=194 2024-12-02T14:21:15,485 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:15,488 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=194, resume processing ppid=193 2024-12-02T14:21:15,488 INFO [PEWorker-1 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-02T14:21:15,488 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=194, ppid=193, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 200 msec 2024-12-02T14:21:15,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=193, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 222 msec 2024-12-02T14:21:15,503 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:15,504 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x46f94893 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@48656c7c 2024-12-02T14:21:15,507 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:15,507 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:15,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=193 2024-12-02T14:21:15,588 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:21:15,589 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-02T14:21:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=195, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:21:15,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=195 2024-12-02T14:21:15,592 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:15,593 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x73923e18 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4f1d8073 2024-12-02T14:21:15,597 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x73923e18 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x73923e18 to 127.0.0.1:53873 2024-12-02T14:21:15,597 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:15,601 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=196, ppid=195, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:15,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=195 2024-12-02T14:21:15,753 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=196 2024-12-02T14:21:15,753 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:21:15,754 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:21:15,754 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:21:15,755 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:15,755 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:15,756 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x16896d58 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2dabf03b 2024-12-02T14:21:15,756 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:15,777 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:15,778 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:21:15,778 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=196 2024-12-02T14:21:15,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=196 2024-12-02T14:21:15,780 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:15,782 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=196, resume processing ppid=195 2024-12-02T14:21:15,783 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=196, ppid=195, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 179 msec 2024-12-02T14:21:15,783 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-02T14:21:15,784 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=195, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 194 msec 2024-12-02T14:21:15,793 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:15,794 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6ee3b91d to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3ae3798f 2024-12-02T14:21:15,796 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:15,796 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:15,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=195 2024-12-02T14:21:15,908 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:21:15,909 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:15,910 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-12-02T14:21:15,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=197, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:21:15,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=197 2024-12-02T14:21:15,912 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:15,913 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0212ee86 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3ec91ef4 2024-12-02T14:21:15,916 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x0212ee86 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0212ee86 to 127.0.0.1:53873 2024-12-02T14:21:15,916 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:15,919 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=198, ppid=197, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:16,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=197 2024-12-02T14:21:16,071 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=198 2024-12-02T14:21:16,071 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:21:16,072 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:21:16,072 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:21:16,072 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:16,072 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:16,073 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x24a40fe1 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5a82d4ce 2024-12-02T14:21:16,074 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:16,089 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:16,089 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:21:16,090 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=198 2024-12-02T14:21:16,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=198 2024-12-02T14:21:16,092 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:16,094 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=198, resume processing ppid=197 2024-12-02T14:21:16,094 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-12-02T14:21:16,094 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=198, ppid=197, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 173 msec 2024-12-02T14:21:16,096 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=197, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 184 msec 2024-12-02T14:21:16,108 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:16,110 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2d22d113 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6bb70be5 2024-12-02T14:21:16,112 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:16,112 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:16,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=197 2024-12-02T14:21:16,228 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:21:16,228 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:16,229 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:16,230 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-02T14:21:16,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=199, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:21:16,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=199 2024-12-02T14:21:16,233 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:16,234 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2f837fff to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2489ded8 2024-12-02T14:21:16,236 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x2f837fff {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2f837fff to 127.0.0.1:53873 2024-12-02T14:21:16,236 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:16,239 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=200, ppid=199, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:16,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=199 2024-12-02T14:21:16,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=200 2024-12-02T14:21:16,391 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:21:16,392 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:21:16,392 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:21:16,392 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:16,392 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:16,394 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x27fbbf7c to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2f96ba09 2024-12-02T14:21:16,394 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:16,415 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:16,415 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:21:16,415 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=200 2024-12-02T14:21:16,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=200 2024-12-02T14:21:16,418 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:16,419 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=200, resume processing ppid=199 2024-12-02T14:21:16,419 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-02T14:21:16,419 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=200, ppid=199, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 179 msec 2024-12-02T14:21:16,420 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=199, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 189 msec 2024-12-02T14:21:16,428 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:16,429 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x66c4ef44 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2f51206 2024-12-02T14:21:16,431 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:16,431 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:16,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=199 2024-12-02T14:21:16,548 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:21:16,548 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:16,549 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:16,550 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-02T14:21:16,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=201, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:21:16,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=201 2024-12-02T14:21:16,553 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:16,554 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4d3fc90f to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5e7e1206 2024-12-02T14:21:16,557 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x4d3fc90f {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4d3fc90f to 127.0.0.1:53873 2024-12-02T14:21:16,558 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:16,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=202, ppid=201, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:16,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=201 2024-12-02T14:21:16,714 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=202 2024-12-02T14:21:16,714 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:21:16,715 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:21:16,715 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:21:16,715 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:16,716 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:16,717 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3fbb21b5 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@34377b89 2024-12-02T14:21:16,717 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:16,737 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:16,738 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:21:16,738 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=202 2024-12-02T14:21:16,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=202 2024-12-02T14:21:16,741 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:16,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=202, resume processing ppid=201 2024-12-02T14:21:16,742 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=202, ppid=201, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 180 msec 2024-12-02T14:21:16,742 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-02T14:21:16,743 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=201, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 192 msec 2024-12-02T14:21:16,752 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:16,753 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0018a0f2 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2ed6a9ee 2024-12-02T14:21:16,755 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:16,755 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:16,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=201 2024-12-02T14:21:16,869 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:21:16,869 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:16,870 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:16,871 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-02T14:21:16,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=203, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:21:16,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=203 2024-12-02T14:21:16,874 DEBUG [PEWorker-5 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:16,875 DEBUG [PEWorker-5 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2f9948fb to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5e5ccbaf 2024-12-02T14:21:16,877 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x2f9948fb {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2f9948fb to 127.0.0.1:53873 2024-12-02T14:21:16,877 WARN [PEWorker-5 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:16,879 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=204, ppid=203, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:16,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=203 2024-12-02T14:21:17,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=204 2024-12-02T14:21:17,032 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:21:17,033 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:21:17,033 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:21:17,033 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:17,033 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:17,034 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x17b1fd01 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4e465d1a 2024-12-02T14:21:17,035 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:17,049 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:17,049 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:21:17,049 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=204 2024-12-02T14:21:17,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=204 2024-12-02T14:21:17,051 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:17,053 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=204, resume processing ppid=203 2024-12-02T14:21:17,053 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=204, ppid=203, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 172 msec 2024-12-02T14:21:17,053 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-02T14:21:17,054 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=203, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 182 msec 2024-12-02T14:21:17,063 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:17,064 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6411cc8e to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6c3bf257 2024-12-02T14:21:17,066 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:17,066 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=203 2024-12-02T14:21:17,188 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:21:17,189 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:17,190 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:17,190 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-02T14:21:17,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=205, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:21:17,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=205 2024-12-02T14:21:17,193 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:17,193 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x707b6ef6 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@44bd968 2024-12-02T14:21:17,197 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x707b6ef6 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x707b6ef6 to 127.0.0.1:53873 2024-12-02T14:21:17,197 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:17,200 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=206, ppid=205, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:17,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=205 2024-12-02T14:21:17,352 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=206 2024-12-02T14:21:17,352 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:21:17,353 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:21:17,353 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:21:17,354 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:17,354 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:17,355 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6d5b8f16 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@51e7c 2024-12-02T14:21:17,356 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:17,371 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:17,371 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:21:17,371 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=206 2024-12-02T14:21:17,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=206 2024-12-02T14:21:17,374 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:17,376 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=206, resume processing ppid=205 2024-12-02T14:21:17,376 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-02T14:21:17,376 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=206, ppid=205, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 175 msec 2024-12-02T14:21:17,378 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=205, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 186 msec 2024-12-02T14:21:17,391 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:17,392 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0d9f12ed to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@466f867a 2024-12-02T14:21:17,394 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:17,395 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:17,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=205 2024-12-02T14:21:17,508 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:21:17,509 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:17,510 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-02T14:21:17,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=207, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:21:17,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=207 2024-12-02T14:21:17,512 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:17,513 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x61183b07 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@296ed025 2024-12-02T14:21:17,516 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x61183b07 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x61183b07 to 127.0.0.1:53873 2024-12-02T14:21:17,516 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:17,518 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=208, ppid=207, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:17,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=207 2024-12-02T14:21:17,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=208 2024-12-02T14:21:17,671 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:21:17,672 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=208 2024-12-02T14:21:17,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=208 2024-12-02T14:21:17,674 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:17,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=208, resume processing ppid=207 2024-12-02T14:21:17,676 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=208, ppid=207, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 156 msec 2024-12-02T14:21:17,676 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-02T14:21:17,677 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=207, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 166 msec 2024-12-02T14:21:17,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=207 2024-12-02T14:21:17,828 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:21:17,828 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:17,829 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:17,830 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-12-02T14:21:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=209, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:21:17,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=209 2024-12-02T14:21:17,833 DEBUG [PEWorker-3 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:17,834 DEBUG [PEWorker-3 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x32399e08 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1d6d7028 2024-12-02T14:21:17,836 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x32399e08 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x32399e08 to 127.0.0.1:53873 2024-12-02T14:21:17,836 WARN [PEWorker-3 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:17,839 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=210, ppid=209, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:17,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=209 2024-12-02T14:21:17,991 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=210 2024-12-02T14:21:17,991 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:21:17,992 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:21:17,992 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:21:17,992 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:17,992 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:17,994 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x49186945 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@14519580 2024-12-02T14:21:17,995 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:18,010 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:18,010 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:21:18,011 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=210 2024-12-02T14:21:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=210 2024-12-02T14:21:18,014 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:18,015 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=210, resume processing ppid=209 2024-12-02T14:21:18,015 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-12-02T14:21:18,015 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=210, ppid=209, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 175 msec 2024-12-02T14:21:18,016 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=209, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 185 msec 2024-12-02T14:21:18,031 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:18,032 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x55eff098 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@25a19f4a 2024-12-02T14:21:18,034 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:18,034 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=209 2024-12-02T14:21:18,148 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:21:18,148 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:18,149 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-12-02T14:21:18,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=211, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:21:18,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=211 2024-12-02T14:21:18,152 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:18,153 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x288d25b8 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5d20911 2024-12-02T14:21:18,156 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x288d25b8 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x288d25b8 to 127.0.0.1:53873 2024-12-02T14:21:18,156 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:18,159 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=212, ppid=211, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:18,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=211 2024-12-02T14:21:18,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=212 2024-12-02T14:21:18,312 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:21:18,313 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:21:18,313 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:21:18,313 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:18,313 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:18,315 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3c6888f7 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@378884cf 2024-12-02T14:21:18,315 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:18,336 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:18,336 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:21:18,336 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=212 2024-12-02T14:21:18,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=212 2024-12-02T14:21:18,338 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:18,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=212, resume processing ppid=211 2024-12-02T14:21:18,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=212, ppid=211, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 179 msec 2024-12-02T14:21:18,340 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-12-02T14:21:18,341 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=211, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 191 msec 2024-12-02T14:21:18,350 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:18,351 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6b7b3362 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4354a787 2024-12-02T14:21:18,353 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:18,353 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:18,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=211 2024-12-02T14:21:18,468 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:21:18,469 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:18,470 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:21:18,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=213, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:18,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=213 2024-12-02T14:21:18,473 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:21:18,473 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-02T14:21:18,474 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:21:18,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:21:18,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:21:18,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-02T14:21:18,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:21:18,475 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:21:18,476 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=214, ppid=213, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=213 2024-12-02T14:21:18,628 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=214 2024-12-02T14:21:18,628 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-02T14:21:18,628 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-02T14:21:18,628 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Replication stream was removed by a user 2024-12-02T14:21:18,629 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:18,629 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:18,630 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4338cfe9 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7a70827d 2024-12-02T14:21:18,630 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:18,632 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSourceManager(562): Done with the queue 1-a3a61c9ba14f,46013,1733149239111 2024-12-02T14:21:18,638 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=214 2024-12-02T14:21:18,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=214 2024-12-02T14:21:18,640 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:18,642 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=214, resume processing ppid=213 2024-12-02T14:21:18,642 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-02T14:21:18,642 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=214, ppid=213, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 164 msec 2024-12-02T14:21:18,642 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-02T14:21:18,648 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-02T14:21:18,650 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=213, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 178 msec 2024-12-02T14:21:18,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=213 2024-12-02T14:21:18,788 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:21:18,789 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:21:18,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=215, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:18,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=215 2024-12-02T14:21:18,791 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:18,792 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=215, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:18,792 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=215, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-02T14:21:18,793 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:18,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=215 2024-12-02T14:21:18,898 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-02T14:21:18,899 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:21:18,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=216, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=216 2024-12-02T14:21:18,902 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:18,903 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=216, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:18,904 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=216, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-02T14:21:18,904 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:19,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=216 2024-12-02T14:21:19,008 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:21:19,026 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[1] Thread=565 (was 513) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x6d5b8f16 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x6b7b3362-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0018a0f2-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x6b7b3362 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x6b7b3362-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x49186945-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x3fbb21b5-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x6ee3b91d-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x4338cfe9-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x4338cfe9-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x66c4ef44 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x27fbbf7c-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x46f94893-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x16896d58-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x55eff098-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x49186945 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x2d22d113-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x24a40fe1-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x2d22d113-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0018a0f2-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x3fbb21b5-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x17b1fd01-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x16896d58-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x6d5b8f16-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x17b1fd01 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x6ee3b91d java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x6ee3b91d-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x55eff098-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x6d5b8f16-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x49186945-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0d9f12ed-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x6411cc8e java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x6411cc8e-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x27fbbf7c java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x55eff098 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x6411cc8e-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x2d22d113 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x3c6888f7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x17b1fd01-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0d9f12ed-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x24a40fe1-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x27fbbf7c-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x3c6888f7-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x66c4ef44-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x46f94893-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x66c4ef44-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x3fbb21b5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0018a0f2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x16896d58 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x3c6888f7-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x46f94893 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x24a40fe1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x0d9f12ed java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x4338cfe9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=903 (was 833) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=422 (was 433), ProcessCount=11 (was 11), AvailableMemoryMB=5702 (was 5716) 2024-12-02T14:21:19,027 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=565 is superior to 500 2024-12-02T14:21:19,038 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[1] Thread=565, OpenFileDescriptor=903, MaxFileDescriptor=1048576, SystemLoadAverage=422, ProcessCount=11, AvailableMemoryMB=5700 2024-12-02T14:21:19,039 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=565 is superior to 500 2024-12-02T14:21:19,039 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:21:19,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=217, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:21:19,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=217 2024-12-02T14:21:19,042 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:19,043 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4f97ea58 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@122b467c 2024-12-02T14:21:19,046 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x4f97ea58 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4f97ea58 to 127.0.0.1:53873 2024-12-02T14:21:19,046 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:19,054 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=218, ppid=217, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:19,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=217 2024-12-02T14:21:19,208 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=218 2024-12-02T14:21:19,208 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=218}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-02T14:21:19,246 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=218}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:19,249 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=218}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=218 2024-12-02T14:21:19,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=218 2024-12-02T14:21:19,251 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:19,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=218, resume processing ppid=217 2024-12-02T14:21:19,253 INFO [PEWorker-4 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-02T14:21:19,253 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=218, ppid=217, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 197 msec 2024-12-02T14:21:19,254 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=217, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 213 msec 2024-12-02T14:21:19,264 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:19,265 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x240d7163 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5c0fd1c8 2024-12-02T14:21:19,267 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:19,267 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:19,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=217 2024-12-02T14:21:19,358 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:21:19,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-02T14:21:19,360 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3992): Client=jenkins//172.17.0.2 disable replication peer, id=1 2024-12-02T14:21:19,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=219, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure 2024-12-02T14:21:19,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=219 2024-12-02T14:21:19,364 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=220, ppid=219, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:19,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=219 2024-12-02T14:21:19,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=220 2024-12-02T14:21:19,516 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=DISABLE_PEER 2024-12-02T14:21:19,517 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.ReplicationSourceManager(490): Terminate replication source for 1 2024-12-02T14:21:19,517 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-02T14:21:19,517 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:19,517 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:19,519 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5086ced9 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4e4ed452 2024-12-02T14:21:19,519 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:19,539 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:19,540 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.ReplicationSourceManager(500): Startup replication source for 1 2024-12-02T14:21:19,540 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=220 2024-12-02T14:21:19,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=220 2024-12-02T14:21:19,542 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for DISABLE on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:19,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=220, resume processing ppid=219 2024-12-02T14:21:19,545 INFO [PEWorker-1 {}] replication.DisablePeerProcedure(67): Successfully disabled peer 1 2024-12-02T14:21:19,545 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=220, ppid=219, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 178 msec 2024-12-02T14:21:19,546 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=219, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure in 185 msec 2024-12-02T14:21:19,556 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:19,557 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7bb6c908 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@747337b1 2024-12-02T14:21:19,559 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:19,559 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:19,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=219 2024-12-02T14:21:19,678 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: DISABLE_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:21:19,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-02T14:21:19,679 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:21:19,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=221, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:19,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=221 2024-12-02T14:21:19,683 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:21:19,683 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-02T14:21:19,683 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:21:19,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:21:19,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:21:19,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-02T14:21:19,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:21:19,686 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:21:19,687 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=222, ppid=221, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:19,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=221 2024-12-02T14:21:19,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=222 2024-12-02T14:21:19,840 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-02T14:21:19,840 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-02T14:21:19,840 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Replication stream was removed by a user 2024-12-02T14:21:19,840 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:19,840 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:19,841 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x67175443 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@14b07c14 2024-12-02T14:21:19,842 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:19,842 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSourceManager(562): Done with the queue 1-a3a61c9ba14f,46013,1733149239111 2024-12-02T14:21:19,844 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=222 2024-12-02T14:21:19,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=222 2024-12-02T14:21:19,846 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:19,848 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=222, resume processing ppid=221 2024-12-02T14:21:19,848 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-02T14:21:19,848 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=222, ppid=221, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 159 msec 2024-12-02T14:21:19,848 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-02T14:21:19,854 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-02T14:21:19,855 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=221, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 175 msec 2024-12-02T14:21:19,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=221 2024-12-02T14:21:19,999 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:21:19,999 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:21:20,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=223, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:20,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=223 2024-12-02T14:21:20,002 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:20,002 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=223, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:20,003 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=223, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-02T14:21:20,003 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:20,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=223 2024-12-02T14:21:20,109 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-02T14:21:20,109 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:21:20,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=224, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:20,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=224 2024-12-02T14:21:20,111 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:20,112 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=224, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:20,113 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=224, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-02T14:21:20,113 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:20,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=224 2024-12-02T14:21:20,219 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:21:20,237 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[1] Thread=576 (was 565) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x5086ced9-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x5086ced9-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x7bb6c908-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x5086ced9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x7bb6c908-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x240d7163-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x240d7163 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x7bb6c908 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x67175443-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x67175443 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x67175443-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x240d7163-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) - Thread LEAK? -, OpenFileDescriptor=919 (was 903) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=422 (was 422), ProcessCount=11 (was 11), AvailableMemoryMB=5697 (was 5700) 2024-12-02T14:21:20,237 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=576 is superior to 500 2024-12-02T14:21:20,251 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[1] Thread=576, OpenFileDescriptor=919, MaxFileDescriptor=1048576, SystemLoadAverage=422, ProcessCount=11, AvailableMemoryMB=5696 2024-12-02T14:21:20,251 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=576 is superior to 500 2024-12-02T14:21:20,251 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=dummy_peer 2024-12-02T14:21:20,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=225, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:20,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=225 2024-12-02T14:21:20,253 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer dummy_peer, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:20,254 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=225, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:20,255 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=225, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-02T14:21:20,255 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:20,343 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T14:21:20,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=225 2024-12-02T14:21:20,358 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: dummy_peer failed with dummy_peer 2024-12-02T14:21:20,359 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:21:20,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=226, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:20,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=226 2024-12-02T14:21:20,362 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:20,363 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=226, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:20,364 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=226, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-02T14:21:20,364 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=226 2024-12-02T14:21:20,468 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-02T14:21:20,468 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:21:20,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=227, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:20,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=227 2024-12-02T14:21:20,471 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:20,472 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=227, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:20,473 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=227, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-02T14:21:20,473 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:20,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=227 2024-12-02T14:21:20,578 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:21:20,597 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[1] Thread=577 (was 576) - Thread LEAK? -, OpenFileDescriptor=919 (was 919), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=422 (was 422), ProcessCount=11 (was 11), AvailableMemoryMB=5695 (was 5696) 2024-12-02T14:21:20,597 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=577 is superior to 500 2024-12-02T14:21:20,612 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerConfig[1] Thread=577, OpenFileDescriptor=919, MaxFileDescriptor=1048576, SystemLoadAverage=422, ProcessCount=11, AvailableMemoryMB=5695 2024-12-02T14:21:20,612 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=577 is superior to 500 2024-12-02T14:21:20,613 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:21:20,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=228, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:21:20,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=228 2024-12-02T14:21:20,616 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:20,618 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x38c7c614 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4fe48231 2024-12-02T14:21:20,629 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x38c7c614 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x38c7c614 to 127.0.0.1:53873 2024-12-02T14:21:20,630 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:20,639 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=229, ppid=228, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:20,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=228 2024-12-02T14:21:20,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=229 2024-12-02T14:21:20,791 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=229}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-02T14:21:20,831 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=229}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:20,833 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=229}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=229 2024-12-02T14:21:20,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=229 2024-12-02T14:21:20,836 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:20,838 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=229, resume processing ppid=228 2024-12-02T14:21:20,838 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=229, ppid=228, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 198 msec 2024-12-02T14:21:20,838 INFO [PEWorker-5 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-02T14:21:20,842 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=228, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 226 msec 2024-12-02T14:21:20,857 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:20,858 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3a307040 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@bd393c1 2024-12-02T14:21:20,869 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:20,869 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=228 2024-12-02T14:21:20,938 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:21:20,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4025): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-02T14:21:20,939 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:21:20,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=230, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:20,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=230 2024-12-02T14:21:20,945 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:21:20,945 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-02T14:21:20,946 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:21:20,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:21:20,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:21:20,947 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-02T14:21:20,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:21:20,948 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:21:20,949 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=231, ppid=230, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:21,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=230 2024-12-02T14:21:21,101 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=231 2024-12-02T14:21:21,101 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-02T14:21:21,101 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-02T14:21:21,101 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Replication stream was removed by a user 2024-12-02T14:21:21,101 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:21,101 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:21,103 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x40e82378 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6a6d864e 2024-12-02T14:21:21,103 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:21,103 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.ReplicationSourceManager(562): Done with the queue 1-a3a61c9ba14f,46013,1733149239111 2024-12-02T14:21:21,106 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=231 2024-12-02T14:21:21,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=231 2024-12-02T14:21:21,108 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:21,110 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=231, resume processing ppid=230 2024-12-02T14:21:21,110 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-02T14:21:21,110 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=231, ppid=230, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 159 msec 2024-12-02T14:21:21,110 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-02T14:21:21,116 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-02T14:21:21,118 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=230, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 176 msec 2024-12-02T14:21:21,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=230 2024-12-02T14:21:21,258 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:21:21,259 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:21:21,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=232, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:21,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=232 2024-12-02T14:21:21,261 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:21,262 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=232, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:21,263 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=232, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-02T14:21:21,263 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:21,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=232 2024-12-02T14:21:21,369 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-02T14:21:21,369 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:21:21,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=233, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:21,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=233 2024-12-02T14:21:21,371 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:21,372 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=233, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:21,373 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=233, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-02T14:21:21,373 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:21,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=233 2024-12-02T14:21:21,479 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:21:21,502 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerConfig[1] Thread=582 (was 577) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x3a307040-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x40e82378-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x40e82378-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x3a307040-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x3a307040 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x40e82378 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=925 (was 919) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=437 (was 422) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=5688 (was 5695) 2024-12-02T14:21:21,502 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=582 is superior to 500 2024-12-02T14:21:21,518 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[1] Thread=582, OpenFileDescriptor=925, MaxFileDescriptor=1048576, SystemLoadAverage=437, ProcessCount=11, AvailableMemoryMB=5688 2024-12-02T14:21:21,518 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=582 is superior to 500 2024-12-02T14:21:21,521 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:21:21,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:21,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] ipc.CallRunner(138): callId: 681 service: MasterService methodName: AddReplicationPeer size: 90 connection: 172.17.0.2:51456 deadline: 1733149341521, exception=java.io.IOException: Replication peer modification disabled 2024-12-02T14:21:21,522 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to master failed, tries = 1, maxAttempts = 2, timeout = 120000 ms, time elapsed = 1 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:465) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:21:21,628 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:21:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:21,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] ipc.CallRunner(138): callId: 682 service: MasterService methodName: AddReplicationPeer size: 90 connection: 172.17.0.2:51456 deadline: 1733149341628, exception=java.io.IOException: Replication peer modification disabled 2024-12-02T14:21:21,629 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to master failed, tries = 2, maxAttempts = 2, timeout = 120000 ms, time elapsed = 108 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:465) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-12-02T14:21:21,630 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Failed after attempts=2, exceptions: 2024-12-02T14:21:21.522Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:21.630Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3963) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3975) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2098) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-12-02T14:21:21,632 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:21:21,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=234, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:21,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=234 2024-12-02T14:21:21,634 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:21,635 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=234, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:21,637 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=234, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-02T14:21:21,637 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:21,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=234 2024-12-02T14:21:21,738 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-02T14:21:21,739 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:21:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=235, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=235 2024-12-02T14:21:21,740 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:21,742 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=235, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:21,743 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=235, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-02T14:21:21,743 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:21,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=235 2024-12-02T14:21:21,848 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:21:21,867 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[1] Thread=582 (was 582), OpenFileDescriptor=925 (was 925), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=437 (was 437), ProcessCount=11 (was 11), AvailableMemoryMB=5687 (was 5688) 2024-12-02T14:21:21,868 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=582 is superior to 500 2024-12-02T14:21:21,884 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[1] Thread=582, OpenFileDescriptor=925, MaxFileDescriptor=1048576, SystemLoadAverage=437, ProcessCount=11, AvailableMemoryMB=5687 2024-12-02T14:21:21,884 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=582 is superior to 500 2024-12-02T14:21:21,885 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:21:21,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=236, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:21:21,889 DEBUG [PEWorker-2 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=236 2024-12-02T14:21:21,890 DEBUG [PEWorker-2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x77f302d9 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1d2b2598 2024-12-02T14:21:21,894 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x77f302d9 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x77f302d9 to 127.0.0.1:53873 2024-12-02T14:21:21,894 WARN [PEWorker-2 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:21,901 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=237, ppid=236, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:21,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=236 2024-12-02T14:21:22,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=237 2024-12-02T14:21:22,054 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=237}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-02T14:21:22,079 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=237}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:22,082 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=237}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=237 2024-12-02T14:21:22,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=237 2024-12-02T14:21:22,084 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:22,085 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=237, resume processing ppid=236 2024-12-02T14:21:22,085 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=237, ppid=236, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 183 msec 2024-12-02T14:21:22,085 INFO [PEWorker-1 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-02T14:21:22,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=236, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 200 msec 2024-12-02T14:21:22,095 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:22,096 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x73272f5e to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@27f66aef 2024-12-02T14:21:22,098 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:22,098 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:22,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=236 2024-12-02T14:21:22,210 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:21:22,211 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:22,213 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:21:22,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=238, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:22,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=238 2024-12-02T14:21:22,218 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:21:22,218 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-02T14:21:22,219 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:21:22,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:21:22,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:21:22,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-02T14:21:22,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:21:22,221 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:21:22,221 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=239, ppid=238, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:22,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=238 2024-12-02T14:21:22,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=239 2024-12-02T14:21:22,374 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-02T14:21:22,374 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-02T14:21:22,374 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Replication stream was removed by a user 2024-12-02T14:21:22,375 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:22,375 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:22,376 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7af83a13 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@72c6be13 2024-12-02T14:21:22,377 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:22,377 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.ReplicationSourceManager(562): Done with the queue 1-a3a61c9ba14f,46013,1733149239111 2024-12-02T14:21:22,380 INFO [master/a3a61c9ba14f:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-02T14:21:22,380 INFO [master/a3a61c9ba14f:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-02T14:21:22,381 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=239 2024-12-02T14:21:22,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=239 2024-12-02T14:21:22,385 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:22,387 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=239, resume processing ppid=238 2024-12-02T14:21:22,387 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=239, ppid=238, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 164 msec 2024-12-02T14:21:22,387 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-02T14:21:22,387 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-02T14:21:22,392 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-02T14:21:22,393 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=238, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 179 msec 2024-12-02T14:21:22,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=238 2024-12-02T14:21:22,529 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:21:22,531 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:21:22,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=240, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:22,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=240 2024-12-02T14:21:22,535 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:22,536 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=240, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:22,538 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=240, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-02T14:21:22,538 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:22,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=240 2024-12-02T14:21:22,639 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:21:22,657 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[1] Thread=588 (was 582) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x7af83a13 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x73272f5e-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x7af83a13-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x73272f5e-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x7af83a13-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x73272f5e java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=933 (was 925) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=437 (was 437), ProcessCount=11 (was 11), AvailableMemoryMB=5686 (was 5687) 2024-12-02T14:21:22,657 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=588 is superior to 500 2024-12-02T14:21:22,672 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerBandwidth[1] Thread=588, OpenFileDescriptor=933, MaxFileDescriptor=1048576, SystemLoadAverage=437, ProcessCount=11, AvailableMemoryMB=5685 2024-12-02T14:21:22,672 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=588 is superior to 500 2024-12-02T14:21:22,673 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:21:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=241, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:21:22,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=241 2024-12-02T14:21:22,675 DEBUG [PEWorker-4 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:22,676 DEBUG [PEWorker-4 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4e9a1963 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@39c32a44 2024-12-02T14:21:22,679 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x4e9a1963 {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4e9a1963 to 127.0.0.1:53873 2024-12-02T14:21:22,679 WARN [PEWorker-4 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:22,690 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=242, ppid=241, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:22,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=241 2024-12-02T14:21:22,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=242 2024-12-02T14:21:22,844 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=242}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-02T14:21:22,877 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=242}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:22,879 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=242}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=242 2024-12-02T14:21:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=242 2024-12-02T14:21:22,881 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:22,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=242, resume processing ppid=241 2024-12-02T14:21:22,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=242, ppid=241, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 191 msec 2024-12-02T14:21:22,883 INFO [PEWorker-3 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-02T14:21:22,884 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=241, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 210 msec 2024-12-02T14:21:22,891 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:22,892 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1480f5e9 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@316c8013 2024-12-02T14:21:22,894 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 1 more 2024-12-02T14:21:22,894 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(595): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-02T14:21:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=241 2024-12-02T14:21:22,989 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:21:22,990 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:22,992 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4014): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-12-02T14:21:22,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=243, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-02T14:21:22,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=243 2024-12-02T14:21:22,997 DEBUG [PEWorker-1 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:22,999 DEBUG [PEWorker-1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x79568d7b to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@334fde0c 2024-12-02T14:21:23,002 DEBUG [ReadOnlyZKClient-127.0.0.1:53873@0x79568d7b {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x79568d7b to 127.0.0.1:53873 2024-12-02T14:21:23,003 WARN [PEWorker-1 {}] replication.ReplicationPeerManager(450): Can't get peerClusterId for clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1 java.io.IOException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.hadoop.hbase.util.FutureUtils.rethrow(FutureUtils.java:172) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:186) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:335) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:181) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:434) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:234) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-02T14:21:23,006 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=244, ppid=243, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:23,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=243 2024-12-02T14:21:23,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=244 2024-12-02T14:21:23,160 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=244}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-02T14:21:23,162 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=244}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=244 2024-12-02T14:21:23,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=244 2024-12-02T14:21:23,167 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for UPDATE_CONFIG on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:23,172 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=244, resume processing ppid=243 2024-12-02T14:21:23,172 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=hbase+zk://127.0.0.1:53873/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-12-02T14:21:23,172 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=244, ppid=243, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 162 msec 2024-12-02T14:21:23,174 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=243, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 180 msec 2024-12-02T14:21:23,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=243 2024-12-02T14:21:23,319 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-02T14:21:23,321 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4002): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-02T14:21:23,323 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:21:23,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=245, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:23,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=245 2024-12-02T14:21:23,327 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:21:23,327 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-02T14:21:23,328 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:21:23,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:21:23,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:21:23,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-02T14:21:23,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:21:23,331 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:21:23,332 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=246, ppid=245, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:23,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=245 2024-12-02T14:21:23,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=246 2024-12-02T14:21:23,485 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-02T14:21:23,485 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-02T14:21:23,485 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Replication stream was removed by a user 2024-12-02T14:21:23,485 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(543): peerId=1, Interrupted while sleeping between retries 2024-12-02T14:21:23,485 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] client.ZKConnectionRegistryURIFactory(41): connect to hbase cluster with zk quorum='127.0.0.1:53873' and parent='/hbase-test1' 2024-12-02T14:21:23,487 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1632e075 to 127.0.0.1:53873 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1dca9103 2024-12-02T14:21:23,487 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] replication.HBaseReplicationEndpoint(156): Failed to get cluster id for cluster java.io.InterruptedIOException: null at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:184) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:44) ~[classes/:?] at org.apache.hadoop.hbase.client.ClusterConnectionFactory.createAsyncClusterConnection(ClusterConnectionFactory.java:83) ~[classes/:?] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.createConnection(HBaseReplicationEndpoint.java:96) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.connect(HBaseReplicationEndpoint.java:178) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:150) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:592) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: java.lang.InterruptedException at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:386) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.util.FutureUtils.get(FutureUtils.java:182) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 7 more 2024-12-02T14:21:23,488 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.ReplicationSourceManager(562): Done with the queue 1-a3a61c9ba14f,46013,1733149239111 2024-12-02T14:21:23,490 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=246 2024-12-02T14:21:23,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=246 2024-12-02T14:21:23,492 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:23,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=246, resume processing ppid=245 2024-12-02T14:21:23,494 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-02T14:21:23,494 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=246, ppid=245, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 160 msec 2024-12-02T14:21:23,495 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-02T14:21:23,500 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-02T14:21:23,502 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=245, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 177 msec 2024-12-02T14:21:23,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=245 2024-12-02T14:21:23,638 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:21:23,639 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:21:23,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=247, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:23,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=247 2024-12-02T14:21:23,642 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:23,642 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=247, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:23,643 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=247, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-02T14:21:23,643 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:23,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=247 2024-12-02T14:21:23,749 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-02T14:21:23,750 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:21:23,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=248, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:23,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=248 2024-12-02T14:21:23,752 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:23,754 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=248, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:23,756 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=248, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-02T14:21:23,756 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:23,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=248 2024-12-02T14:21:23,861 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:21:23,882 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerBandwidth[1] Thread=594 (was 588) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x1632e075 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x1632e075-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x1480f5e9-EventThread java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x1632e075-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x1480f5e9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$858/0x00007f1bc4be5778.run(Unknown Source) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ReadOnlyZKClient-127.0.0.1:53873@0x1480f5e9-SendThread(127.0.0.1:53873) java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) - Thread LEAK? -, OpenFileDescriptor=941 (was 933) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=437 (was 437), ProcessCount=11 (was 11), AvailableMemoryMB=5674 (was 5685) 2024-12-02T14:21:23,882 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=594 is superior to 500 2024-12-02T14:21:23,895 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[1] Thread=594, OpenFileDescriptor=941, MaxFileDescriptor=1048576, SystemLoadAverage=437, ProcessCount=11, AvailableMemoryMB=5673 2024-12-02T14:21:23,895 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=594 is superior to 500 2024-12-02T14:21:23,896 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:21:23,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=249, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:21:23,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=249 2024-12-02T14:21:23,906 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=250, ppid=249, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:24,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=249 2024-12-02T14:21:24,059 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=250 2024-12-02T14:21:24,059 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=250}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-02T14:21:24,091 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=250}] regionserver.ReplicationSource(231): queueId=1-a3a61c9ba14f,46013,1733149239111, ReplicationSource: 1, currentBandwidth=0 2024-12-02T14:21:24,093 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=250}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=250 2024-12-02T14:21:24,094 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(613): peerId=1, queueId=1-a3a61c9ba14f,46013,1733149239111 (queues=1) is replicating from cluster=5762d0f7-da11-4ede-9d35-feade635cfff to cluster=5762d0f7-da11-4ede-9d35-feade635cfff 2024-12-02T14:21:24,094 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSource(382): peerId=1, starting shipping worker for walGroupId=a3a61c9ba14f%2C46013%2C1733149239111 2024-12-02T14:21:24,094 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSourceWALReader(111): peerClusterZnode=1-a3a61c9ba14f,46013,1733149239111, ReplicationSourceWALReaderThread : 1 inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-12-02T14:21:24,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=250 2024-12-02T14:21:24,094 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111.replicationSource.shippera3a61c9ba14f%2C46013%2C1733149239111,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSourceShipper(98): Running ReplicationSourceShipper Thread for wal group: a3a61c9ba14f%2C46013%2C1733149239111 2024-12-02T14:21:24,094 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111.replicationSource.wal-reader.a3a61c9ba14f%2C46013%2C1733149239111,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.WALEntryStream(254): Creating new reader hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/WALs/a3a61c9ba14f,46013,1733149239111/a3a61c9ba14f%2C46013%2C1733149239111.1733149241428, startPosition=0, beingWritten=true 2024-12-02T14:21:24,095 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for ADD on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:24,097 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=250, resume processing ppid=249 2024-12-02T14:21:24,097 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=250, ppid=249, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 190 msec 2024-12-02T14:21:24,097 INFO [PEWorker-5 {}] replication.AddPeerProcedure(130): Successfully added ENABLED peer 1, config clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-02T14:21:24,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=249, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 201 msec 2024-12-02T14:21:24,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=249 2024-12-02T14:21:24,220 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:21:24,222 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3973): Client=jenkins//172.17.0.2 creating replication peer, id=2, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-02T14:21:24,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=251, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-02T14:21:24,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=251 2024-12-02T14:21:24,227 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] ... 12 more 2024-12-02T14:21:24,230 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=251, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:424) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:251) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:201) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:421) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:475) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:172) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:118) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:24,232 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=251, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=9 msec 2024-12-02T14:21:24,232 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:24,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=251 2024-12-02T14:21:24,340 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: ADD_REPLICATION_PEER, peerId: 2 failed with Invalid cluster key: 2024-12-02T14:21:24,341 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-02T14:21:24,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=252, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:24,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252 2024-12-02T14:21:24,348 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:21:24,349 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-02T14:21:24,349 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:21:24,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/new-sync-rep-state 2024-12-02T14:21:24,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:21:24,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-02T14:21:24,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/sync-rep-state 2024-12-02T14:21:24,353 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-02T14:21:24,353 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=253, ppid=252, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-02T14:21:24,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252 2024-12-02T14:21:24,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=46013 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=253 2024-12-02T14:21:24,506 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=253}] regionserver.RefreshPeerCallable(47): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-02T14:21:24,506 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=253}] regionserver.ReplicationSourceManager(294): Number of deleted recovered sources for 1: 0 2024-12-02T14:21:24,507 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=253}] regionserver.ReplicationSource(687): peerId=1, Closing source 1-a3a61c9ba14f,46013,1733149239111 because: Replication stream was removed by a user 2024-12-02T14:21:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252 2024-12-02T14:21:24,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252 2024-12-02T14:21:25,345 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T14:21:25,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252 2024-12-02T14:21:25,507 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111.replicationSource.shippera3a61c9ba14f%2C46013%2C1733149239111,1-a3a61c9ba14f,46013,1733149239111 {}] regionserver.ReplicationSourceShipper(123): Interrupted while waiting for next replication entry batch java.lang.InterruptedException: null at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1681) ~[?:?] at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) ~[?:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.poll(ReplicationSourceWALReader.java:309) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceShipper.run(ReplicationSourceShipper.java:109) ~[classes/:?] 2024-12-02T14:21:25,507 WARN [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111.replicationSource.wal-reader.a3a61c9ba14f%2C46013%2C1733149239111,1-a3a61c9ba14f,46013,1733149239111 {}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:130) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:162) ~[classes/:?] 2024-12-02T14:21:26,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252 2024-12-02T14:21:26,507 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=253}] regionserver.ReplicationSource(739): peerId=1, ReplicationSourceWorker RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0.replicationSource,1-a3a61c9ba14f,46013,1733149239111.replicationSource.shippera3a61c9ba14f%2C46013%2C1733149239111,1-a3a61c9ba14f,46013,1733149239111 terminated 2024-12-02T14:21:26,507 INFO [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=253}] regionserver.ReplicationSourceManager(562): Done with the queue 1-a3a61c9ba14f,46013,1733149239111 2024-12-02T14:21:26,513 DEBUG [RS_REFRESH_PEER-regionserver/a3a61c9ba14f:0-0 {event_type=RS_REFRESH_PEER, pid=253}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=253 2024-12-02T14:21:26,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(4169): Remote procedure done, pid=253 2024-12-02T14:21:26,519 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(132): Refresh peer 1 for REMOVE on a3a61c9ba14f,46013,1733149239111 suceeded 2024-12-02T14:21:26,522 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=253, resume processing ppid=252 2024-12-02T14:21:26,522 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(89): No ongoing assign replication queues procedures when removing peer 1, move on 2024-12-02T14:21:26,522 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=253, ppid=252, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 2.1660 sec 2024-12-02T14:21:26,522 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(98): All assign replication queues procedures are finished when removing peer 1, move on 2024-12-02T14:21:26,528 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(120): Successfully removed peer 1 2024-12-02T14:21:26,530 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=252, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 2.1870 sec 2024-12-02T14:21:28,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=252 2024-12-02T14:21:28,509 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3004): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-02T14:21:28,510 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.HMaster(3980): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-02T14:21:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] procedure2.ProcedureExecutor(1139): Stored pid=254, state=RUNNABLE:PRE_PEER_MODIFICATION, hasLock=false; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-02T14:21:28,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=254 2024-12-02T14:21:28,514 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(190): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] 2024-12-02T14:21:28,515 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1697): Root Procedure pid=254, state=FAILED, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:765) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:45) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:941) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1825) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1503) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2154) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2181) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:192) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:210) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:70) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:188) ~[classes/:?] ... 8 more 2024-12-02T14:21:28,517 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1707): Rolled back pid=254, state=ROLLEDBACK, hasLock=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-02T14:21:28,517 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2061): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-02T14:21:28,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36883 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=254 2024-12-02T14:21:28,620 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(3009): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-02T14:21:28,648 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[1] Thread=596 (was 594) - Thread LEAK? -, OpenFileDescriptor=946 (was 941) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=410 (was 437), ProcessCount=11 (was 11), AvailableMemoryMB=5668 (was 5673) 2024-12-02T14:21:28,648 WARN [Time-limited test {}] hbase.ResourceChecker(130): Thread=596 is superior to 500 2024-12-02T14:21:28,648 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T14:21:28,648 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.client.TestAsyncAdminBase.tearDownAfterClass(TestAsyncAdminBase.java:96) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T14:21:28,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:28,649 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:28,649 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:21:28,649 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-12-02T14:21:28,650 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T14:21:28,650 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestAsyncAdminBase.tearDownAfterClass(TestAsyncAdminBase.java:97) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T14:21:28,650 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:28,650 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:28,650 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-12-02T14:21:28,650 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-02T14:21:28,650 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=870052201, stopped=false 2024-12-02T14:21:28,651 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=a3a61c9ba14f,36883,1733149238328 2024-12-02T14:21:28,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T14:21:28,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46013-0x1009b4dfd050001, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-02T14:21:28,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:21:28,652 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46013-0x1009b4dfd050001, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:21:28,652 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T14:21:28,652 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-12-02T14:21:28,652 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.client.TestAsyncAdminBase.tearDownAfterClass(TestAsyncAdminBase.java:97) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T14:21:28,652 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:28,652 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T14:21:28,652 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:46013-0x1009b4dfd050001, quorum=127.0.0.1:53873, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-02T14:21:28,653 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'a3a61c9ba14f,46013,1733149239111' ***** 2024-12-02T14:21:28,653 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-12-02T14:21:28,653 INFO [RS:0;a3a61c9ba14f:46013 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-02T14:21:28,653 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-12-02T14:21:28,654 INFO [RS:0;a3a61c9ba14f:46013 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-02T14:21:28,654 INFO [RS:0;a3a61c9ba14f:46013 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-02T14:21:28,654 INFO [RS:0;a3a61c9ba14f:46013 {}] regionserver.HRegionServer(3091): Received CLOSE for 3edf40fb8fe820e5620b9e92d8c85956 2024-12-02T14:21:28,654 INFO [RS:0;a3a61c9ba14f:46013 {}] regionserver.HRegionServer(959): stopping server a3a61c9ba14f,46013,1733149239111 2024-12-02T14:21:28,655 INFO [RS:0;a3a61c9ba14f:46013 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T14:21:28,655 INFO [RS:0;a3a61c9ba14f:46013 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;a3a61c9ba14f:46013. 2024-12-02T14:21:28,655 DEBUG [RS:0;a3a61c9ba14f:46013 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-02T14:21:28,655 DEBUG [RS:0;a3a61c9ba14f:46013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:28,655 INFO [RS:0;a3a61c9ba14f:46013 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-02T14:21:28,655 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1722): Closing 3edf40fb8fe820e5620b9e92d8c85956, disabling compactions & flushes 2024-12-02T14:21:28,655 INFO [RS:0;a3a61c9ba14f:46013 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-02T14:21:28,655 INFO [RS:0;a3a61c9ba14f:46013 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-02T14:21:28,655 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1755): Closing region hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956. 2024-12-02T14:21:28,655 INFO [RS:0;a3a61c9ba14f:46013 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-12-02T14:21:28,655 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956. 2024-12-02T14:21:28,655 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1843): Acquired close lock on hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956. after waiting 0 ms 2024-12-02T14:21:28,655 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1853): Updates disabled for region hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956. 2024-12-02T14:21:28,655 INFO [RS:0;a3a61c9ba14f:46013 {}] regionserver.HRegionServer(1321): Waiting on 2 regions to close 2024-12-02T14:21:28,655 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-12-02T14:21:28,655 DEBUG [RS:0;a3a61c9ba14f:46013 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740, 3edf40fb8fe820e5620b9e92d8c85956=hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956.} 2024-12-02T14:21:28,655 INFO [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-12-02T14:21:28,656 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-12-02T14:21:28,656 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-02T14:21:28,656 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-12-02T14:21:28,656 DEBUG [RS:0;a3a61c9ba14f:46013 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3edf40fb8fe820e5620b9e92d8c85956 2024-12-02T14:21:28,660 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2902): Flushing 3edf40fb8fe820e5620b9e92d8c85956 3/3 column families, dataSize=5.05 KB heapSize=10.72 KB 2024-12-02T14:21:28,660 INFO [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=1.34 KB heapSize=3.38 KB 2024-12-02T14:21:28,696 INFO [regionserver/a3a61c9ba14f:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T14:21:28,723 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/replication/3edf40fb8fe820e5620b9e92d8c85956/.tmp/hfileref/5916b47a6e584156934c36735374d773 is 33, key is 1/hfileref:/1733149286527/DeleteFamily/seqid=0 2024-12-02T14:21:28,723 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/meta/1588230740/.tmp/info/65359a7054244bffaf714b527b5eb8ae is 147, key is hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956./info:regioninfo/1733149243073/Put/seqid=0 2024-12-02T14:21:28,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37975 is added to blk_1073741839_1015 (size=6631) 2024-12-02T14:21:28,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37975 is added to blk_1073741838_1014 (size=5733) 2024-12-02T14:21:28,856 DEBUG [RS:0;a3a61c9ba14f:46013 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3edf40fb8fe820e5620b9e92d8c85956 2024-12-02T14:21:29,056 DEBUG [RS:0;a3a61c9ba14f:46013 {}] regionserver.HRegionServer(1351): Waiting on 1588230740, 3edf40fb8fe820e5620b9e92d8c85956 2024-12-02T14:21:29,137 INFO [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.17 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/meta/1588230740/.tmp/info/65359a7054244bffaf714b527b5eb8ae 2024-12-02T14:21:29,137 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=638 B at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/replication/3edf40fb8fe820e5620b9e92d8c85956/.tmp/hfileref/5916b47a6e584156934c36735374d773 2024-12-02T14:21:29,170 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5916b47a6e584156934c36735374d773 2024-12-02T14:21:29,197 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/meta/1588230740/.tmp/ns/c900a554140f4d17b61852c4f1434e9e is 43, key is default/ns:d/1733149242232/Put/seqid=0 2024-12-02T14:21:29,200 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/replication/3edf40fb8fe820e5620b9e92d8c85956/.tmp/queue/25820823e95747b698067a9fed45958c is 63, key is 1-a3a61c9ba14f,46013,1733149239111/queue:/1733149286510/DeleteFamily/seqid=0 2024-12-02T14:21:29,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37975 is added to blk_1073741840_1016 (size=5153) 2024-12-02T14:21:29,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37975 is added to blk_1073741841_1017 (size=6582) 2024-12-02T14:21:29,205 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.43 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/replication/3edf40fb8fe820e5620b9e92d8c85956/.tmp/queue/25820823e95747b698067a9fed45958c 2024-12-02T14:21:29,213 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 25820823e95747b698067a9fed45958c 2024-12-02T14:21:29,215 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/replication/3edf40fb8fe820e5620b9e92d8c85956/.tmp/hfileref/5916b47a6e584156934c36735374d773 as hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/replication/3edf40fb8fe820e5620b9e92d8c85956/hfileref/5916b47a6e584156934c36735374d773 2024-12-02T14:21:29,223 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 5916b47a6e584156934c36735374d773 2024-12-02T14:21:29,223 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/replication/3edf40fb8fe820e5620b9e92d8c85956/hfileref/5916b47a6e584156934c36735374d773, entries=22, sequenceid=70, filesize=5.6 K 2024-12-02T14:21:29,226 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/replication/3edf40fb8fe820e5620b9e92d8c85956/.tmp/queue/25820823e95747b698067a9fed45958c as hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/replication/3edf40fb8fe820e5620b9e92d8c85956/queue/25820823e95747b698067a9fed45958c 2024-12-02T14:21:29,234 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 25820823e95747b698067a9fed45958c 2024-12-02T14:21:29,234 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/replication/3edf40fb8fe820e5620b9e92d8c85956/queue/25820823e95747b698067a9fed45958c, entries=22, sequenceid=70, filesize=6.4 K 2024-12-02T14:21:29,241 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3140): Finished flush of dataSize ~5.05 KB/5170, heapSize ~10.44 KB/10688, currentSize=0 B/0 for 3edf40fb8fe820e5620b9e92d8c85956 in 579ms, sequenceid=70, compaction requested=false 2024-12-02T14:21:29,248 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/replication/3edf40fb8fe820e5620b9e92d8c85956/recovered.edits/73.seqid, newMaxSeqId=73, maxSeqId=1 2024-12-02T14:21:29,248 INFO [regionserver/a3a61c9ba14f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-02T14:21:29,248 INFO [regionserver/a3a61c9ba14f:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-02T14:21:29,251 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T14:21:29,251 INFO [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1973): Closed hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956. 2024-12-02T14:21:29,251 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1676): Region close journal for 3edf40fb8fe820e5620b9e92d8c85956: Waiting for close lock at 1733149288655Running coprocessor pre-close hooks at 1733149288655Disabling compacts and flushes for region at 1733149288655Disabling writes for close at 1733149288655Obtaining lock to block concurrent updates at 1733149288660 (+5 ms)Preparing flush snapshotting stores in 3edf40fb8fe820e5620b9e92d8c85956 at 1733149288660Finished memstore snapshotting hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956., syncing WAL and waiting on mvcc, flushsize=dataSize=5170, getHeapSize=10928, getOffHeapSize=0, getCellsCount=66 at 1733149288668 (+8 ms)Flushing stores of hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956. at 1733149288669 (+1 ms)Flushing 3edf40fb8fe820e5620b9e92d8c85956/hfileref: creating writer at 1733149288672 (+3 ms)Flushing 3edf40fb8fe820e5620b9e92d8c85956/hfileref: appending metadata at 1733149288711 (+39 ms)Flushing 3edf40fb8fe820e5620b9e92d8c85956/hfileref: closing flushed file at 1733149288713 (+2 ms)Flushing 3edf40fb8fe820e5620b9e92d8c85956/queue: creating writer at 1733149289182 (+469 ms)Flushing 3edf40fb8fe820e5620b9e92d8c85956/queue: appending metadata at 1733149289197 (+15 ms)Flushing 3edf40fb8fe820e5620b9e92d8c85956/queue: closing flushed file at 1733149289197Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3cfc3d70: reopening flushed file at 1733149289213 (+16 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@142998d3: reopening flushed file at 1733149289225 (+12 ms)Finished flush of dataSize ~5.05 KB/5170, heapSize ~10.44 KB/10688, currentSize=0 B/0 for 3edf40fb8fe820e5620b9e92d8c85956 in 579ms, sequenceid=70, compaction requested=false at 1733149289241 (+16 ms)Writing region close event to WAL at 1733149289243 (+2 ms)Running coprocessor post-close hooks at 1733149289249 (+6 ms)Closed at 1733149289251 (+2 ms) 2024-12-02T14:21:29,252 DEBUG [RS_CLOSE_REGION-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:replication,,1733149242578.3edf40fb8fe820e5620b9e92d8c85956. 2024-12-02T14:21:29,257 DEBUG [RS:0;a3a61c9ba14f:46013 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T14:21:29,457 DEBUG [RS:0;a3a61c9ba14f:46013 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T14:21:29,607 INFO [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=74 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/meta/1588230740/.tmp/ns/c900a554140f4d17b61852c4f1434e9e 2024-12-02T14:21:29,637 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/meta/1588230740/.tmp/table/7ebfe516d6cb477d8771aa5e043fecbb is 53, key is hbase:replication/table:state/1733149243096/Put/seqid=0 2024-12-02T14:21:29,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37975 is added to blk_1073741842_1018 (size=5256) 2024-12-02T14:21:29,657 INFO [RS:0;a3a61c9ba14f:46013 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-12-02T14:21:29,657 DEBUG [RS:0;a3a61c9ba14f:46013 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-12-02T14:21:29,657 DEBUG [RS:0;a3a61c9ba14f:46013 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T14:21:29,858 DEBUG [RS:0;a3a61c9ba14f:46013 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T14:21:30,044 INFO [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=98 B at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/meta/1588230740/.tmp/table/7ebfe516d6cb477d8771aa5e043fecbb 2024-12-02T14:21:30,058 DEBUG [RS:0;a3a61c9ba14f:46013 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-12-02T14:21:30,059 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/meta/1588230740/.tmp/info/65359a7054244bffaf714b527b5eb8ae as hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/meta/1588230740/info/65359a7054244bffaf714b527b5eb8ae 2024-12-02T14:21:30,069 INFO [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/meta/1588230740/info/65359a7054244bffaf714b527b5eb8ae, entries=10, sequenceid=11, filesize=6.5 K 2024-12-02T14:21:30,071 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/meta/1588230740/.tmp/ns/c900a554140f4d17b61852c4f1434e9e as hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/meta/1588230740/ns/c900a554140f4d17b61852c4f1434e9e 2024-12-02T14:21:30,079 INFO [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/meta/1588230740/ns/c900a554140f4d17b61852c4f1434e9e, entries=2, sequenceid=11, filesize=5.0 K 2024-12-02T14:21:30,081 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/meta/1588230740/.tmp/table/7ebfe516d6cb477d8771aa5e043fecbb as hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/meta/1588230740/table/7ebfe516d6cb477d8771aa5e043fecbb 2024-12-02T14:21:30,089 INFO [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/meta/1588230740/table/7ebfe516d6cb477d8771aa5e043fecbb, entries=2, sequenceid=11, filesize=5.1 K 2024-12-02T14:21:30,090 INFO [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~1.34 KB/1368, heapSize ~3.08 KB/3152, currentSize=0 B/0 for 1588230740 in 1434ms, sequenceid=11, compaction requested=false 2024-12-02T14:21:30,095 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/data/hbase/meta/1588230740/recovered.edits/14.seqid, newMaxSeqId=14, maxSeqId=1 2024-12-02T14:21:30,096 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-02T14:21:30,096 INFO [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-12-02T14:21:30,096 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1733149288655Running coprocessor pre-close hooks at 1733149288655Disabling compacts and flushes for region at 1733149288655Disabling writes for close at 1733149288656 (+1 ms)Obtaining lock to block concurrent updates at 1733149288660 (+4 ms)Preparing flush snapshotting stores in 1588230740 at 1733149288660Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=1368, getHeapSize=3392, getOffHeapSize=0, getCellsCount=14 at 1733149288668 (+8 ms)Flushing stores of hbase:meta,,1.1588230740 at 1733149288669 (+1 ms)Flushing 1588230740/info: creating writer at 1733149288672 (+3 ms)Flushing 1588230740/info: appending metadata at 1733149288711 (+39 ms)Flushing 1588230740/info: closing flushed file at 1733149288713 (+2 ms)Flushing 1588230740/ns: creating writer at 1733149289182 (+469 ms)Flushing 1588230740/ns: appending metadata at 1733149289197 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1733149289197Flushing 1588230740/table: creating writer at 1733149289621 (+424 ms)Flushing 1588230740/table: appending metadata at 1733149289636 (+15 ms)Flushing 1588230740/table: closing flushed file at 1733149289636Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ff0d334: reopening flushed file at 1733149290058 (+422 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@51e2fa7: reopening flushed file at 1733149290069 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@21db234: reopening flushed file at 1733149290080 (+11 ms)Finished flush of dataSize ~1.34 KB/1368, heapSize ~3.08 KB/3152, currentSize=0 B/0 for 1588230740 in 1434ms, sequenceid=11, compaction requested=false at 1733149290090 (+10 ms)Writing region close event to WAL at 1733149290091 (+1 ms)Running coprocessor post-close hooks at 1733149290096 (+5 ms)Closed at 1733149290096 2024-12-02T14:21:30,097 DEBUG [RS_CLOSE_META-regionserver/a3a61c9ba14f:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-02T14:21:30,258 INFO [RS:0;a3a61c9ba14f:46013 {}] regionserver.HRegionServer(976): stopping server a3a61c9ba14f,46013,1733149239111; all regions closed. 2024-12-02T14:21:30,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37975 is added to blk_1073741834_1010 (size=2742) 2024-12-02T14:21:30,270 DEBUG [RS:0;a3a61c9ba14f:46013 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/oldWALs 2024-12-02T14:21:30,270 INFO [RS:0;a3a61c9ba14f:46013 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a3a61c9ba14f%2C46013%2C1733149239111.meta:.meta(num 1733149242045) 2024-12-02T14:21:30,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37975 is added to blk_1073741837_1013 (size=11128) 2024-12-02T14:21:30,276 DEBUG [RS:0;a3a61c9ba14f:46013 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/oldWALs 2024-12-02T14:21:30,276 INFO [RS:0;a3a61c9ba14f:46013 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a3a61c9ba14f%2C46013%2C1733149239111.rep:(num 1733149243005) 2024-12-02T14:21:30,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37975 is added to blk_1073741832_1008 (size=93) 2024-12-02T14:21:30,281 DEBUG [RS:0;a3a61c9ba14f:46013 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/oldWALs 2024-12-02T14:21:30,281 INFO [RS:0;a3a61c9ba14f:46013 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL a3a61c9ba14f%2C46013%2C1733149239111:(num 1733149241428) 2024-12-02T14:21:30,282 DEBUG [RS:0;a3a61c9ba14f:46013 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-02T14:21:30,282 INFO [RS:0;a3a61c9ba14f:46013 {}] regionserver.LeaseManager(133): Closed leases 2024-12-02T14:21:30,282 INFO [RS:0;a3a61c9ba14f:46013 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T14:21:30,282 INFO [RS:0;a3a61c9ba14f:46013 {}] hbase.ChoreService(370): Chore service for: regionserver/a3a61c9ba14f:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-12-02T14:21:30,282 INFO [RS:0;a3a61c9ba14f:46013 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T14:21:30,282 INFO [regionserver/a3a61c9ba14f:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T14:21:30,283 INFO [RS:0;a3a61c9ba14f:46013 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:46013 2024-12-02T14:21:30,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46013-0x1009b4dfd050001, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/a3a61c9ba14f,46013,1733149239111 2024-12-02T14:21:30,287 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-02T14:21:30,287 INFO [RS:0;a3a61c9ba14f:46013 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T14:21:30,288 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [a3a61c9ba14f,46013,1733149239111] 2024-12-02T14:21:30,289 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/a3a61c9ba14f,46013,1733149239111 already deleted, retry=false 2024-12-02T14:21:30,289 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; a3a61c9ba14f,46013,1733149239111 expired; onlineServers=0 2024-12-02T14:21:30,289 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'a3a61c9ba14f,36883,1733149238328' ***** 2024-12-02T14:21:30,289 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-02T14:21:30,289 INFO [M:0;a3a61c9ba14f:36883 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-12-02T14:21:30,289 INFO [M:0;a3a61c9ba14f:36883 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-12-02T14:21:30,290 DEBUG [M:0;a3a61c9ba14f:36883 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-02T14:21:30,290 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-02T14:21:30,290 DEBUG [M:0;a3a61c9ba14f:36883 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-02T14:21:30,290 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster-HFileCleaner.large.0-1733149241099 {}] cleaner.HFileCleaner(306): Exit Thread[master/a3a61c9ba14f:0:becomeActiveMaster-HFileCleaner.large.0-1733149241099,5,FailOnTimeoutGroup] 2024-12-02T14:21:30,290 DEBUG [master/a3a61c9ba14f:0:becomeActiveMaster-HFileCleaner.small.0-1733149241102 {}] cleaner.HFileCleaner(306): Exit Thread[master/a3a61c9ba14f:0:becomeActiveMaster-HFileCleaner.small.0-1733149241102,5,FailOnTimeoutGroup] 2024-12-02T14:21:30,290 INFO [M:0;a3a61c9ba14f:36883 {}] hbase.ChoreService(370): Chore service for: master/a3a61c9ba14f:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-12-02T14:21:30,290 INFO [M:0;a3a61c9ba14f:36883 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-12-02T14:21:30,290 DEBUG [M:0;a3a61c9ba14f:36883 {}] master.HMaster(1795): Stopping service threads 2024-12-02T14:21:30,290 INFO [M:0;a3a61c9ba14f:36883 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-02T14:21:30,290 INFO [M:0;a3a61c9ba14f:36883 {}] procedure2.ProcedureExecutor(723): Stopping 2024-12-02T14:21:30,291 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-02T14:21:30,291 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-02T14:21:30,291 INFO [M:0;a3a61c9ba14f:36883 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-02T14:21:30,291 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-02T14:21:30,291 DEBUG [M:0;a3a61c9ba14f:36883 {}] zookeeper.ZKUtil(347): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-02T14:21:30,291 WARN [M:0;a3a61c9ba14f:36883 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-02T14:21:30,293 INFO [M:0;a3a61c9ba14f:36883 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/.lastflushedseqids 2024-12-02T14:21:30,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37975 is added to blk_1073741843_1019 (size=172) 2024-12-02T14:21:30,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46013-0x1009b4dfd050001, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T14:21:30,388 INFO [RS:0;a3a61c9ba14f:46013 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T14:21:30,388 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:46013-0x1009b4dfd050001, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T14:21:30,389 INFO [RS:0;a3a61c9ba14f:46013 {}] regionserver.HRegionServer(1031): Exiting; stopping=a3a61c9ba14f,46013,1733149239111; zookeeper connection closed. 2024-12-02T14:21:30,389 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@1c5c4a0e {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@1c5c4a0e 2024-12-02T14:21:30,390 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-02T14:21:30,707 INFO [M:0;a3a61c9ba14f:36883 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-12-02T14:21:30,708 INFO [M:0;a3a61c9ba14f:36883 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-02T14:21:30,708 DEBUG [M:0;a3a61c9ba14f:36883 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-02T14:21:30,708 INFO [M:0;a3a61c9ba14f:36883 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T14:21:30,708 DEBUG [M:0;a3a61c9ba14f:36883 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T14:21:30,708 DEBUG [M:0;a3a61c9ba14f:36883 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-02T14:21:30,709 DEBUG [M:0;a3a61c9ba14f:36883 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T14:21:30,709 INFO [M:0;a3a61c9ba14f:36883 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=752.47 KB heapSize=898.47 KB 2024-12-02T14:21:30,729 DEBUG [M:0;a3a61c9ba14f:36883 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/db6731ecf1b54e75874ca8d15c6b5c79 is 82, key is hbase:meta,,1/info:regioninfo/1733149242154/Put/seqid=0 2024-12-02T14:21:30,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37975 is added to blk_1073741844_1020 (size=5672) 2024-12-02T14:21:31,136 INFO [M:0;a3a61c9ba14f:36883 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=1773 (bloomFilter=true), to=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/db6731ecf1b54e75874ca8d15c6b5c79 2024-12-02T14:21:31,177 DEBUG [M:0;a3a61c9ba14f:36883 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f3005087ac2e487596ab8bb3f1f41697 is 2004, key is \x00\x00\x00\x00\x00\x00\x00\x9B/proc:d/1733149269818/Put/seqid=0 2024-12-02T14:21:31,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37975 is added to blk_1073741845_1021 (size=163415) 2024-12-02T14:21:31,581 INFO [M:0;a3a61c9ba14f:36883 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=749.91 KB at sequenceid=1773 (bloomFilter=true), to=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f3005087ac2e487596ab8bb3f1f41697 2024-12-02T14:21:31,594 INFO [M:0;a3a61c9ba14f:36883 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f3005087ac2e487596ab8bb3f1f41697 2024-12-02T14:21:31,606 DEBUG [M:0;a3a61c9ba14f:36883 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/eacf6edcc73f440a85dd87cdb490b0a5 is 69, key is a3a61c9ba14f,46013,1733149239111/rs:state/1733149241147/Put/seqid=0 2024-12-02T14:21:31,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37975 is added to blk_1073741846_1022 (size=5156) 2024-12-02T14:21:32,012 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-02T14:21:32,014 INFO [M:0;a3a61c9ba14f:36883 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=1773 (bloomFilter=true), to=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/eacf6edcc73f440a85dd87cdb490b0a5 2024-12-02T14:21:32,042 DEBUG [M:0;a3a61c9ba14f:36883 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1d036d105a5046359e0af4879d2c7d1c is 68, key is replication_peer_modification_on/state:d/1733149288627/Put/seqid=0 2024-12-02T14:21:32,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37975 is added to blk_1073741847_1023 (size=5154) 2024-12-02T14:21:32,449 INFO [M:0;a3a61c9ba14f:36883 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2 KB at sequenceid=1773 (bloomFilter=true), to=hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1d036d105a5046359e0af4879d2c7d1c 2024-12-02T14:21:32,465 DEBUG [M:0;a3a61c9ba14f:36883 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/db6731ecf1b54e75874ca8d15c6b5c79 as hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/db6731ecf1b54e75874ca8d15c6b5c79 2024-12-02T14:21:32,473 INFO [M:0;a3a61c9ba14f:36883 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/db6731ecf1b54e75874ca8d15c6b5c79, entries=8, sequenceid=1773, filesize=5.5 K 2024-12-02T14:21:32,474 DEBUG [M:0;a3a61c9ba14f:36883 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f3005087ac2e487596ab8bb3f1f41697 as hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f3005087ac2e487596ab8bb3f1f41697 2024-12-02T14:21:32,482 INFO [M:0;a3a61c9ba14f:36883 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f3005087ac2e487596ab8bb3f1f41697 2024-12-02T14:21:32,482 INFO [M:0;a3a61c9ba14f:36883 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f3005087ac2e487596ab8bb3f1f41697, entries=254, sequenceid=1773, filesize=159.6 K 2024-12-02T14:21:32,484 DEBUG [M:0;a3a61c9ba14f:36883 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/eacf6edcc73f440a85dd87cdb490b0a5 as hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/eacf6edcc73f440a85dd87cdb490b0a5 2024-12-02T14:21:32,491 INFO [M:0;a3a61c9ba14f:36883 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/eacf6edcc73f440a85dd87cdb490b0a5, entries=1, sequenceid=1773, filesize=5.0 K 2024-12-02T14:21:32,493 DEBUG [M:0;a3a61c9ba14f:36883 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/1d036d105a5046359e0af4879d2c7d1c as hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/1d036d105a5046359e0af4879d2c7d1c 2024-12-02T14:21:32,501 INFO [M:0;a3a61c9ba14f:36883 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:46667/user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/1d036d105a5046359e0af4879d2c7d1c, entries=1, sequenceid=1773, filesize=5.0 K 2024-12-02T14:21:32,502 INFO [M:0;a3a61c9ba14f:36883 {}] regionserver.HRegion(3140): Finished flush of dataSize ~752.47 KB/770527, heapSize ~898.41 KB/919968, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1793ms, sequenceid=1773, compaction requested=false 2024-12-02T14:21:32,504 INFO [M:0;a3a61c9ba14f:36883 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-02T14:21:32,504 DEBUG [M:0;a3a61c9ba14f:36883 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1733149290708Disabling compacts and flushes for region at 1733149290708Disabling writes for close at 1733149290708Obtaining lock to block concurrent updates at 1733149290709 (+1 ms)Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1733149290709Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=770527, getHeapSize=919968, getOffHeapSize=0, getCellsCount=1964 at 1733149290710 (+1 ms)Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1733149290712 (+2 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1733149290712Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1733149290729 (+17 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1733149290729Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1733149291150 (+421 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1733149291177 (+27 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1733149291177Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1733149291594 (+417 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1733149291606 (+12 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1733149291606Flushing 1595e783b53d99cd5eef43b6debb2682/state: creating writer at 1733149292027 (+421 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: appending metadata at 1733149292041 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/state: closing flushed file at 1733149292041Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@bfbaedd: reopening flushed file at 1733149292463 (+422 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5e3db493: reopening flushed file at 1733149292473 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@312b3dca: reopening flushed file at 1733149292482 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@995661a: reopening flushed file at 1733149292492 (+10 ms)Finished flush of dataSize ~752.47 KB/770527, heapSize ~898.41 KB/919968, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1793ms, sequenceid=1773, compaction requested=false at 1733149292502 (+10 ms)Writing region close event to WAL at 1733149292504 (+2 ms)Closed at 1733149292504 2024-12-02T14:21:32,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37975 is added to blk_1073741830_1006 (size=886989) 2024-12-02T14:21:32,507 WARN [Close-WAL-Writer-0 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(650): complete file /user/jenkins/test-data/549810c7-7966-9c5a-3520-eb9e2fb8ebc4/MasterData/WALs/a3a61c9ba14f,36883,1733149238328/a3a61c9ba14f%2C36883%2C1733149238328.1733149240466 not finished, retry = 0 2024-12-02T14:21:32,610 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-12-02T14:21:32,610 INFO [M:0;a3a61c9ba14f:36883 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-12-02T14:21:32,611 INFO [M:0;a3a61c9ba14f:36883 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:36883 2024-12-02T14:21:32,611 INFO [M:0;a3a61c9ba14f:36883 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-12-02T14:21:32,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T14:21:32,714 INFO [M:0;a3a61c9ba14f:36883 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-12-02T14:21:32,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36883-0x1009b4dfd050000, quorum=127.0.0.1:53873, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-02T14:21:32,722 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@30338fac{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-02T14:21:32,725 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@7f8511bb{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T14:21:32,726 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T14:21:32,726 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@43c2f64f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T14:21:32,726 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@39bd1973{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/hadoop.log.dir/,STOPPED} 2024-12-02T14:21:32,730 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-02T14:21:32,730 WARN [BP-1636995057-172.17.0.2-1733149234894 heartbeating to localhost/127.0.0.1:46667 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-02T14:21:32,730 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-02T14:21:32,730 WARN [BP-1636995057-172.17.0.2-1733149234894 heartbeating to localhost/127.0.0.1:46667 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1636995057-172.17.0.2-1733149234894 (Datanode Uuid 168ed01c-ffa7-4f2a-95c0-899bedb822af) service to localhost/127.0.0.1:46667 2024-12-02T14:21:32,732 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/cluster_c3337032-8624-86a2-153d-f1cb3af65c88/data/data1/current/BP-1636995057-172.17.0.2-1733149234894 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T14:21:32,732 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/cluster_c3337032-8624-86a2-153d-f1cb3af65c88/data/data2/current/BP-1636995057-172.17.0.2-1733149234894 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-02T14:21:32,732 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-02T14:21:32,738 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1441d181{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-02T14:21:32,739 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@135deabe{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-02T14:21:32,739 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-02T14:21:32,739 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7b133317{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-02T14:21:32,739 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@33fb8453{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/acc92ff8-3a1f-f3a2-f1f5-dfdd94fe4226/hadoop.log.dir/,STOPPED} 2024-12-02T14:21:32,751 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-12-02T14:21:32,767 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down