2024-11-11 03:17:08,226 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@302c971f 2024-11-11 03:17:08,241 main DEBUG Took 0.012615 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-11 03:17:08,241 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-11 03:17:08,242 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-11 03:17:08,243 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-11 03:17:08,245 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 03:17:08,264 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-11 03:17:08,279 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:17:08,281 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 03:17:08,282 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:17:08,282 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 03:17:08,283 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:17:08,283 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 03:17:08,285 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:17:08,285 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 03:17:08,286 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:17:08,286 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 03:17:08,287 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:17:08,287 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 03:17:08,288 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:17:08,289 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 03:17:08,289 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:17:08,290 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 03:17:08,290 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:17:08,291 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 03:17:08,291 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:17:08,292 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 03:17:08,292 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:17:08,293 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 03:17:08,293 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:17:08,294 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-11 03:17:08,294 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:17:08,295 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-11 03:17:08,297 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-11 03:17:08,298 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-11 03:17:08,300 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-11 03:17:08,301 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-11 03:17:08,302 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-11 03:17:08,303 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-11 03:17:08,314 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-11 03:17:08,318 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-11 03:17:08,320 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-11 03:17:08,320 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-11 03:17:08,321 main DEBUG createAppenders(={Console}) 2024-11-11 03:17:08,322 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@302c971f initialized 2024-11-11 03:17:08,322 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@302c971f 2024-11-11 03:17:08,322 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@302c971f OK. 2024-11-11 03:17:08,323 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-11 03:17:08,324 main DEBUG OutputStream closed 2024-11-11 03:17:08,324 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-11 03:17:08,324 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-11 03:17:08,325 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@505fc5a4 OK 2024-11-11 03:17:08,412 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-11 03:17:08,415 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-11 03:17:08,417 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-11 03:17:08,418 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-11 03:17:08,419 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-11 03:17:08,419 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-11 03:17:08,420 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-11 03:17:08,420 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-11 03:17:08,421 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-11 03:17:08,421 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-11 03:17:08,422 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-11 03:17:08,422 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-11 03:17:08,423 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-11 03:17:08,423 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-11 03:17:08,424 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-11 03:17:08,424 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-11 03:17:08,424 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-11 03:17:08,425 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-11 03:17:08,427 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11 03:17:08,428 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@39de3d36) with optional ClassLoader: null 2024-11-11 03:17:08,428 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-11 03:17:08,429 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@39de3d36] started OK. 2024-11-11T03:17:08,720 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221 2024-11-11 03:17:08,724 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-11 03:17:08,724 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-11T03:17:08,754 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestAsyncReplicationAdminApi timeout: 26 mins 2024-11-11T03:17:08,778 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-11T03:17:08,782 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/cluster_ed1c06ec-b623-8234-3e18-7586117bca1f, deleteOnExit=true 2024-11-11T03:17:08,782 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-11-11T03:17:08,783 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/test.cache.data in system properties and HBase conf 2024-11-11T03:17:08,783 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/hadoop.tmp.dir in system properties and HBase conf 2024-11-11T03:17:08,784 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/hadoop.log.dir in system properties and HBase conf 2024-11-11T03:17:08,785 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-11T03:17:08,785 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-11T03:17:08,786 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-11-11T03:17:08,890 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-11T03:17:09,006 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-11T03:17:09,011 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-11T03:17:09,012 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-11T03:17:09,012 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-11T03:17:09,013 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T03:17:09,014 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-11T03:17:09,014 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-11T03:17:09,015 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-11T03:17:09,015 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T03:17:09,016 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-11T03:17:09,017 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/nfs.dump.dir in system properties and HBase conf 2024-11-11T03:17:09,017 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/java.io.tmpdir in system properties and HBase conf 2024-11-11T03:17:09,018 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-11T03:17:09,018 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-11T03:17:09,019 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-11T03:17:09,922 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-11T03:17:10,004 INFO [Time-limited test {}] log.Log(170): Logging initialized @2599ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-11T03:17:10,098 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T03:17:10,174 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T03:17:10,205 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T03:17:10,205 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T03:17:10,207 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-11T03:17:10,222 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T03:17:10,224 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@842d13f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/hadoop.log.dir/,AVAILABLE} 2024-11-11T03:17:10,225 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@201ce359{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T03:17:10,439 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2490b214{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/java.io.tmpdir/jetty-localhost-35231-hadoop-hdfs-3_4_1-tests_jar-_-any-11026193757260295615/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T03:17:10,448 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1f7e2804{HTTP/1.1, (http/1.1)}{localhost:35231} 2024-11-11T03:17:10,449 INFO [Time-limited test {}] server.Server(415): Started @3045ms 2024-11-11T03:17:10,854 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-11T03:17:10,861 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-11T03:17:10,862 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-11T03:17:10,863 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-11T03:17:10,863 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-11T03:17:10,864 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5b9056db{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/hadoop.log.dir/,AVAILABLE} 2024-11-11T03:17:10,864 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@50244d8f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-11T03:17:10,983 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@19f1b88e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/java.io.tmpdir/jetty-localhost-39567-hadoop-hdfs-3_4_1-tests_jar-_-any-15100519672592812576/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T03:17:10,984 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2117487{HTTP/1.1, (http/1.1)}{localhost:39567} 2024-11-11T03:17:10,984 INFO [Time-limited test {}] server.Server(415): Started @3581ms 2024-11-11T03:17:11,041 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-11T03:17:11,487 WARN [Thread-71 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/cluster_ed1c06ec-b623-8234-3e18-7586117bca1f/dfs/data/data1/current/BP-669106914-172.17.0.2-1731295029668/current, will proceed with Du for space computation calculation, 2024-11-11T03:17:11,487 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/cluster_ed1c06ec-b623-8234-3e18-7586117bca1f/dfs/data/data2/current/BP-669106914-172.17.0.2-1731295029668/current, will proceed with Du for space computation calculation, 2024-11-11T03:17:11,526 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-11T03:17:11,585 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbb39a050b8be0192 with lease ID 0x82d1c87c10b79b10: Processing first storage report for DS-d8921118-796e-46de-ab00-808a3cfc35de from datanode DatanodeRegistration(127.0.0.1:43237, datanodeUuid=90015427-e251-401a-9d78-c5ff268e48fd, infoPort=42839, infoSecurePort=0, ipcPort=33611, storageInfo=lv=-57;cid=testClusterID;nsid=1867378026;c=1731295029668) 2024-11-11T03:17:11,587 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbb39a050b8be0192 with lease ID 0x82d1c87c10b79b10: from storage DS-d8921118-796e-46de-ab00-808a3cfc35de node DatanodeRegistration(127.0.0.1:43237, datanodeUuid=90015427-e251-401a-9d78-c5ff268e48fd, infoPort=42839, infoSecurePort=0, ipcPort=33611, storageInfo=lv=-57;cid=testClusterID;nsid=1867378026;c=1731295029668), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-11T03:17:11,587 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xbb39a050b8be0192 with lease ID 0x82d1c87c10b79b10: Processing first storage report for DS-f5ed079f-802e-4bba-9b58-d69f234563c3 from datanode DatanodeRegistration(127.0.0.1:43237, datanodeUuid=90015427-e251-401a-9d78-c5ff268e48fd, infoPort=42839, infoSecurePort=0, ipcPort=33611, storageInfo=lv=-57;cid=testClusterID;nsid=1867378026;c=1731295029668) 2024-11-11T03:17:11,587 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xbb39a050b8be0192 with lease ID 0x82d1c87c10b79b10: from storage DS-f5ed079f-802e-4bba-9b58-d69f234563c3 node DatanodeRegistration(127.0.0.1:43237, datanodeUuid=90015427-e251-401a-9d78-c5ff268e48fd, infoPort=42839, infoSecurePort=0, ipcPort=33611, storageInfo=lv=-57;cid=testClusterID;nsid=1867378026;c=1731295029668), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-11T03:17:11,622 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221 2024-11-11T03:17:11,696 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/cluster_ed1c06ec-b623-8234-3e18-7586117bca1f/zookeeper_0, clientPort=54139, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/cluster_ed1c06ec-b623-8234-3e18-7586117bca1f/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/cluster_ed1c06ec-b623-8234-3e18-7586117bca1f/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-11T03:17:11,706 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=54139 2024-11-11T03:17:11,719 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T03:17:11,722 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T03:17:11,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43237 is added to blk_1073741825_1001 (size=7) 2024-11-11T03:17:12,364 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3 with version=8 2024-11-11T03:17:12,365 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/hbase-staging 2024-11-11T03:17:12,503 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-11T03:17:12,893 INFO [Time-limited test {}] client.ConnectionUtils(129): master/e4cdd3a0b883:0 server-side Connection retries=6 2024-11-11T03:17:12,922 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T03:17:12,923 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T03:17:12,923 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T03:17:12,923 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T03:17:12,923 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T03:17:13,139 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T03:17:13,255 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-11T03:17:13,272 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-11T03:17:13,278 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T03:17:13,321 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 12491 (auto-detected) 2024-11-11T03:17:13,323 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-11T03:17:13,378 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:36699 2024-11-11T03:17:13,390 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T03:17:13,394 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T03:17:13,430 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:36699 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:13,510 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:366990x0, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:13,512 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36699-0x101955526e60000 connected 2024-11-11T03:17:13,641 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T03:17:13,645 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T03:17:13,649 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T03:17:13,655 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36699 2024-11-11T03:17:13,665 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36699 2024-11-11T03:17:13,685 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36699 2024-11-11T03:17:13,692 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36699 2024-11-11T03:17:13,694 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36699 2024-11-11T03:17:13,704 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3, hbase.cluster.distributed=false 2024-11-11T03:17:13,878 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/e4cdd3a0b883:0 server-side Connection retries=6 2024-11-11T03:17:13,878 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T03:17:13,879 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-11T03:17:13,879 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-11T03:17:13,879 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-11T03:17:13,879 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-11T03:17:13,883 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-11T03:17:13,886 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-11T03:17:13,888 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:41101 2024-11-11T03:17:13,891 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-11T03:17:13,931 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-11T03:17:13,934 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T03:17:13,939 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T03:17:13,944 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:41101 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:13,968 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:411010x0, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T03:17:13,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:411010x0, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:13,973 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:411010x0, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T03:17:13,974 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:411010x0, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-11T03:17:13,982 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:41101-0x101955526e60001 connected 2024-11-11T03:17:13,991 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=41101 2024-11-11T03:17:13,994 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=41101 2024-11-11T03:17:14,016 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=41101 2024-11-11T03:17:14,024 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=41101 2024-11-11T03:17:14,025 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=41101 2024-11-11T03:17:14,045 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/e4cdd3a0b883,36699,1731295032496 2024-11-11T03:17:14,052 DEBUG [M:0;e4cdd3a0b883:36699 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;e4cdd3a0b883:36699 2024-11-11T03:17:14,058 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T03:17:14,059 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T03:17:14,065 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/e4cdd3a0b883,36699,1731295032496 2024-11-11T03:17:14,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T03:17:14,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-11T03:17:14,105 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:17:14,106 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:17:14,110 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T03:17:14,113 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-11T03:17:14,117 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/e4cdd3a0b883,36699,1731295032496 from backup master directory 2024-11-11T03:17:14,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T03:17:14,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/e4cdd3a0b883,36699,1731295032496 2024-11-11T03:17:14,121 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-11T03:17:14,122 WARN [master/e4cdd3a0b883:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T03:17:14,122 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=e4cdd3a0b883,36699,1731295032496 2024-11-11T03:17:14,125 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-11T03:17:14,127 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-11T03:17:14,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43237 is added to blk_1073741826_1002 (size=42) 2024-11-11T03:17:14,255 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/hbase.id with ID: 454cc11f-9833-4fa8-a013-fffc9bade1b4 2024-11-11T03:17:14,325 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-11T03:17:14,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:17:14,370 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:17:14,432 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43237 is added to blk_1073741827_1003 (size=196) 2024-11-11T03:17:14,466 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-11T03:17:14,469 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-11T03:17:14,516 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:402) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:973) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2470) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:590) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:587) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:14,523 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T03:17:14,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43237 is added to blk_1073741828_1004 (size=1189) 2024-11-11T03:17:14,639 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/data/master/store 2024-11-11T03:17:14,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43237 is added to blk_1073741829_1005 (size=34) 2024-11-11T03:17:14,706 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-11T03:17:14,707 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T03:17:14,717 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T03:17:14,717 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T03:17:14,718 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T03:17:14,718 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T03:17:14,718 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T03:17:14,718 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T03:17:14,719 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-11T03:17:14,732 WARN [master/e4cdd3a0b883:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/data/master/store/.initializing 2024-11-11T03:17:14,732 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/WALs/e4cdd3a0b883,36699,1731295032496 2024-11-11T03:17:14,743 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T03:17:14,782 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e4cdd3a0b883%2C36699%2C1731295032496, suffix=, logDir=hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/WALs/e4cdd3a0b883,36699,1731295032496, archiveDir=hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/oldWALs, maxLogs=10 2024-11-11T03:17:14,821 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/WALs/e4cdd3a0b883,36699,1731295032496/e4cdd3a0b883%2C36699%2C1731295032496.1731295034789, exclude list is [], retry=0 2024-11-11T03:17:14,862 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43237,DS-d8921118-796e-46de-ab00-808a3cfc35de,DISK] 2024-11-11T03:17:14,867 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-11T03:17:14,932 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/WALs/e4cdd3a0b883,36699,1731295032496/e4cdd3a0b883%2C36699%2C1731295032496.1731295034789 2024-11-11T03:17:14,937 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42839:42839)] 2024-11-11T03:17:14,938 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-11T03:17:14,941 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T03:17:14,963 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T03:17:14,968 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T03:17:15,085 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T03:17:15,152 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-11T03:17:15,159 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:17:15,163 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T03:17:15,164 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T03:17:15,170 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-11T03:17:15,170 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:17:15,173 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:17:15,173 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T03:17:15,177 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-11T03:17:15,178 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:17:15,180 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:17:15,180 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-11T03:17:15,187 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-11T03:17:15,188 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:17:15,190 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:17:15,205 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T03:17:15,207 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-11T03:17:15,224 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-11T03:17:15,233 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-11T03:17:15,251 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T03:17:15,254 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61859533, jitterRate=-0.07822112739086151}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-11T03:17:15,262 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-11T03:17:15,264 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-11T03:17:15,336 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2721dbae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:15,412 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-11-11T03:17:15,445 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-11T03:17:15,446 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-11T03:17:15,449 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-11T03:17:15,452 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 2 msec 2024-11-11T03:17:15,461 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 8 msec 2024-11-11T03:17:15,461 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-11T03:17:15,579 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-11T03:17:15,616 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-11T03:17:15,621 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-11-11T03:17:15,626 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-11T03:17:15,632 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-11T03:17:15,635 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-11-11T03:17:15,638 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-11T03:17:15,662 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-11T03:17:15,665 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-11-11T03:17:15,667 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-11T03:17:15,669 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-11-11T03:17:15,690 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-11T03:17:15,696 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-11T03:17:15,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T03:17:15,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-11T03:17:15,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:17:15,703 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:17:15,705 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=e4cdd3a0b883,36699,1731295032496, sessionid=0x101955526e60000, setting cluster-up flag (Was=false) 2024-11-11T03:17:15,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:17:15,735 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:17:15,753 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-11T03:17:15,761 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e4cdd3a0b883,36699,1731295032496 2024-11-11T03:17:15,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:17:15,780 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:17:15,794 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-11T03:17:15,801 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=e4cdd3a0b883,36699,1731295032496 2024-11-11T03:17:15,884 DEBUG [RS:0;e4cdd3a0b883:41101 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;e4cdd3a0b883:41101 2024-11-11T03:17:15,890 INFO [RS:0;e4cdd3a0b883:41101 {}] regionserver.HRegionServer(1008): ClusterId : 454cc11f-9833-4fa8-a013-fffc9bade1b4 2024-11-11T03:17:15,894 DEBUG [RS:0;e4cdd3a0b883:41101 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-11T03:17:15,912 DEBUG [RS:0;e4cdd3a0b883:41101 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-11T03:17:15,912 DEBUG [RS:0;e4cdd3a0b883:41101 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-11T03:17:15,919 DEBUG [RS:0;e4cdd3a0b883:41101 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-11T03:17:15,920 DEBUG [RS:0;e4cdd3a0b883:41101 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30061385, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:15,932 DEBUG [RS:0;e4cdd3a0b883:41101 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@764fce4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e4cdd3a0b883/172.17.0.2:0 2024-11-11T03:17:15,937 INFO [RS:0;e4cdd3a0b883:41101 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-11-11T03:17:15,938 INFO [RS:0;e4cdd3a0b883:41101 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-11-11T03:17:15,938 DEBUG [RS:0;e4cdd3a0b883:41101 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-11-11T03:17:15,941 INFO [RS:0;e4cdd3a0b883:41101 {}] regionserver.HRegionServer(3073): reportForDuty to master=e4cdd3a0b883,36699,1731295032496 with isa=e4cdd3a0b883/172.17.0.2:41101, startcode=1731295033876 2024-11-11T03:17:15,963 DEBUG [RS:0;e4cdd3a0b883:41101 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T03:17:15,989 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-11-11T03:17:15,999 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-11-11T03:17:16,002 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-11T03:17:16,009 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: e4cdd3a0b883,36699,1731295032496 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-11T03:17:16,023 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/e4cdd3a0b883:0, corePoolSize=5, maxPoolSize=5 2024-11-11T03:17:16,024 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/e4cdd3a0b883:0, corePoolSize=5, maxPoolSize=5 2024-11-11T03:17:16,031 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/e4cdd3a0b883:0, corePoolSize=5, maxPoolSize=5 2024-11-11T03:17:16,031 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/e4cdd3a0b883:0, corePoolSize=5, maxPoolSize=5 2024-11-11T03:17:16,031 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/e4cdd3a0b883:0, corePoolSize=10, maxPoolSize=10 2024-11-11T03:17:16,031 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/e4cdd3a0b883:0, corePoolSize=1, maxPoolSize=1 2024-11-11T03:17:16,032 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/e4cdd3a0b883:0, corePoolSize=2, maxPoolSize=2 2024-11-11T03:17:16,032 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/e4cdd3a0b883:0, corePoolSize=1, maxPoolSize=1 2024-11-11T03:17:16,047 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54389, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T03:17:16,054 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-11-11T03:17:16,055 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-11-11T03:17:16,055 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36699 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3280) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:593) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-11T03:17:16,066 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:17:16,066 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-11T03:17:16,084 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731295066084 2024-11-11T03:17:16,087 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-11T03:17:16,088 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-11T03:17:16,090 DEBUG [RS:0;e4cdd3a0b883:41101 {}] regionserver.HRegionServer(3097): Master is not running yet 2024-11-11T03:17:16,091 WARN [RS:0;e4cdd3a0b883:41101 {}] regionserver.HRegionServer(1099): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-11T03:17:16,093 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-11T03:17:16,100 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-11T03:17:16,114 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-11T03:17:16,115 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-11T03:17:16,139 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T03:17:16,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43237 is added to blk_1073741831_1007 (size=1039) 2024-11-11T03:17:16,154 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-11-11T03:17:16,155 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3 2024-11-11T03:17:16,157 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-11T03:17:16,159 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-11T03:17:16,159 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-11T03:17:16,184 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-11T03:17:16,185 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-11T03:17:16,192 INFO [RS:0;e4cdd3a0b883:41101 {}] regionserver.HRegionServer(3073): reportForDuty to master=e4cdd3a0b883,36699,1731295032496 with isa=e4cdd3a0b883/172.17.0.2:41101, startcode=1731295033876 2024-11-11T03:17:16,194 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36699 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3280) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:593) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-11T03:17:16,196 DEBUG [RS:0;e4cdd3a0b883:41101 {}] regionserver.HRegionServer(3097): Master is not running yet 2024-11-11T03:17:16,196 WARN [RS:0;e4cdd3a0b883:41101 {}] regionserver.HRegionServer(1099): reportForDuty failed; sleeping 200 ms and then retrying. 2024-11-11T03:17:16,212 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/e4cdd3a0b883:0:becomeActiveMaster-HFileCleaner.large.0-1731295036186,5,FailOnTimeoutGroup] 2024-11-11T03:17:16,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43237 is added to blk_1073741832_1008 (size=32) 2024-11-11T03:17:16,239 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T03:17:16,240 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/e4cdd3a0b883:0:becomeActiveMaster-HFileCleaner.small.0-1731295036213,5,FailOnTimeoutGroup] 2024-11-11T03:17:16,240 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-11T03:17:16,240 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-11T03:17:16,242 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-11T03:17:16,242 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-11T03:17:16,244 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T03:17:16,249 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T03:17:16,249 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:17:16,251 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T03:17:16,252 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T03:17:16,258 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T03:17:16,259 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:17:16,276 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T03:17:16,277 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T03:17:16,282 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T03:17:16,283 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:17:16,284 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T03:17:16,286 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/data/hbase/meta/1588230740 2024-11-11T03:17:16,287 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/data/hbase/meta/1588230740 2024-11-11T03:17:16,292 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-11T03:17:16,295 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-11T03:17:16,335 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T03:17:16,337 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74730189, jitterRate=0.11356659233570099}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-11T03:17:16,341 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-11T03:17:16,342 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-11-11T03:17:16,342 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-11-11T03:17:16,342 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-11-11T03:17:16,342 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T03:17:16,342 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T03:17:16,365 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-11-11T03:17:16,365 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-11-11T03:17:16,369 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-11-11T03:17:16,370 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-11-11T03:17:16,398 INFO [RS:0;e4cdd3a0b883:41101 {}] regionserver.HRegionServer(3073): reportForDuty to master=e4cdd3a0b883,36699,1731295032496 with isa=e4cdd3a0b883/172.17.0.2:41101, startcode=1731295033876 2024-11-11T03:17:16,401 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36699 {}] master.ServerManager(332): Checking decommissioned status of RegionServer e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:16,405 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36699 {}] master.ServerManager(486): Registering regionserver=e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:16,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-11T03:17:16,424 DEBUG [RS:0;e4cdd3a0b883:41101 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3 2024-11-11T03:17:16,424 DEBUG [RS:0;e4cdd3a0b883:41101 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:35635 2024-11-11T03:17:16,424 DEBUG [RS:0;e4cdd3a0b883:41101 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-11-11T03:17:16,430 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T03:17:16,435 DEBUG [RS:0;e4cdd3a0b883:41101 {}] zookeeper.ZKUtil(111): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:16,436 WARN [RS:0;e4cdd3a0b883:41101 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-11T03:17:16,436 INFO [RS:0;e4cdd3a0b883:41101 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T03:17:16,436 DEBUG [RS:0;e4cdd3a0b883:41101 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/WALs/e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:16,443 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-11T03:17:16,444 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [e4cdd3a0b883,41101,1731295033876] 2024-11-11T03:17:16,449 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-11T03:17:16,477 DEBUG [RS:0;e4cdd3a0b883:41101 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-11-11T03:17:16,507 INFO [RS:0;e4cdd3a0b883:41101 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-11T03:17:16,539 INFO [RS:0;e4cdd3a0b883:41101 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-11T03:17:16,543 INFO [RS:0;e4cdd3a0b883:41101 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-11T03:17:16,543 INFO [RS:0;e4cdd3a0b883:41101 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T03:17:16,546 INFO [RS:0;e4cdd3a0b883:41101 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-11-11T03:17:16,563 INFO [RS:0;e4cdd3a0b883:41101 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-11T03:17:16,563 DEBUG [RS:0;e4cdd3a0b883:41101 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/e4cdd3a0b883:0, corePoolSize=1, maxPoolSize=1 2024-11-11T03:17:16,564 DEBUG [RS:0;e4cdd3a0b883:41101 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/e4cdd3a0b883:0, corePoolSize=1, maxPoolSize=1 2024-11-11T03:17:16,564 DEBUG [RS:0;e4cdd3a0b883:41101 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/e4cdd3a0b883:0, corePoolSize=1, maxPoolSize=1 2024-11-11T03:17:16,564 DEBUG [RS:0;e4cdd3a0b883:41101 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/e4cdd3a0b883:0, corePoolSize=1, maxPoolSize=1 2024-11-11T03:17:16,565 DEBUG [RS:0;e4cdd3a0b883:41101 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/e4cdd3a0b883:0, corePoolSize=1, maxPoolSize=1 2024-11-11T03:17:16,565 DEBUG [RS:0;e4cdd3a0b883:41101 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/e4cdd3a0b883:0, corePoolSize=2, maxPoolSize=2 2024-11-11T03:17:16,566 DEBUG [RS:0;e4cdd3a0b883:41101 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/e4cdd3a0b883:0, corePoolSize=1, maxPoolSize=1 2024-11-11T03:17:16,566 DEBUG [RS:0;e4cdd3a0b883:41101 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/e4cdd3a0b883:0, corePoolSize=1, maxPoolSize=1 2024-11-11T03:17:16,566 DEBUG [RS:0;e4cdd3a0b883:41101 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0, corePoolSize=1, maxPoolSize=1 2024-11-11T03:17:16,567 DEBUG [RS:0;e4cdd3a0b883:41101 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/e4cdd3a0b883:0, corePoolSize=1, maxPoolSize=1 2024-11-11T03:17:16,567 DEBUG [RS:0;e4cdd3a0b883:41101 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/e4cdd3a0b883:0, corePoolSize=1, maxPoolSize=1 2024-11-11T03:17:16,567 DEBUG [RS:0;e4cdd3a0b883:41101 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/e4cdd3a0b883:0, corePoolSize=3, maxPoolSize=3 2024-11-11T03:17:16,567 DEBUG [RS:0;e4cdd3a0b883:41101 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/e4cdd3a0b883:0, corePoolSize=3, maxPoolSize=3 2024-11-11T03:17:16,576 INFO [RS:0;e4cdd3a0b883:41101 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T03:17:16,577 INFO [RS:0;e4cdd3a0b883:41101 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-11T03:17:16,577 INFO [RS:0;e4cdd3a0b883:41101 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-11T03:17:16,577 INFO [RS:0;e4cdd3a0b883:41101 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-11T03:17:16,577 INFO [RS:0;e4cdd3a0b883:41101 {}] hbase.ChoreService(168): Chore ScheduledChore name=e4cdd3a0b883,41101,1731295033876-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T03:17:16,601 WARN [e4cdd3a0b883:36699 {}] assignment.AssignmentManager(2423): No servers available; cannot place 1 unassigned regions. 2024-11-11T03:17:16,615 INFO [RS:0;e4cdd3a0b883:41101 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-11T03:17:16,618 INFO [RS:0;e4cdd3a0b883:41101 {}] hbase.ChoreService(168): Chore ScheduledChore name=e4cdd3a0b883,41101,1731295033876-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T03:17:16,647 INFO [RS:0;e4cdd3a0b883:41101 {}] regionserver.Replication(204): e4cdd3a0b883,41101,1731295033876 started 2024-11-11T03:17:16,648 INFO [RS:0;e4cdd3a0b883:41101 {}] regionserver.HRegionServer(1767): Serving as e4cdd3a0b883,41101,1731295033876, RpcServer on e4cdd3a0b883/172.17.0.2:41101, sessionid=0x101955526e60001 2024-11-11T03:17:16,649 DEBUG [RS:0;e4cdd3a0b883:41101 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-11T03:17:16,649 DEBUG [RS:0;e4cdd3a0b883:41101 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:16,649 DEBUG [RS:0;e4cdd3a0b883:41101 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e4cdd3a0b883,41101,1731295033876' 2024-11-11T03:17:16,649 DEBUG [RS:0;e4cdd3a0b883:41101 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-11T03:17:16,652 DEBUG [RS:0;e4cdd3a0b883:41101 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-11T03:17:16,653 DEBUG [RS:0;e4cdd3a0b883:41101 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-11T03:17:16,653 DEBUG [RS:0;e4cdd3a0b883:41101 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-11T03:17:16,653 DEBUG [RS:0;e4cdd3a0b883:41101 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:16,653 DEBUG [RS:0;e4cdd3a0b883:41101 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'e4cdd3a0b883,41101,1731295033876' 2024-11-11T03:17:16,653 DEBUG [RS:0;e4cdd3a0b883:41101 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-11T03:17:16,654 DEBUG [RS:0;e4cdd3a0b883:41101 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-11T03:17:16,655 DEBUG [RS:0;e4cdd3a0b883:41101 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-11T03:17:16,655 INFO [RS:0;e4cdd3a0b883:41101 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-11T03:17:16,655 INFO [RS:0;e4cdd3a0b883:41101 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-11T03:17:16,763 INFO [RS:0;e4cdd3a0b883:41101 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-11T03:17:16,781 INFO [RS:0;e4cdd3a0b883:41101 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e4cdd3a0b883%2C41101%2C1731295033876, suffix=, logDir=hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/WALs/e4cdd3a0b883,41101,1731295033876, archiveDir=hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/oldWALs, maxLogs=32 2024-11-11T03:17:16,818 DEBUG [RS:0;e4cdd3a0b883:41101 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/WALs/e4cdd3a0b883,41101,1731295033876/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784, exclude list is [], retry=0 2024-11-11T03:17:16,835 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43237,DS-d8921118-796e-46de-ab00-808a3cfc35de,DISK] 2024-11-11T03:17:16,852 INFO [RS:0;e4cdd3a0b883:41101 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/WALs/e4cdd3a0b883,41101,1731295033876/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:17:16,861 DEBUG [RS:0;e4cdd3a0b883:41101 {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42839:42839)] 2024-11-11T03:17:17,104 DEBUG [e4cdd3a0b883:36699 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-11T03:17:17,111 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:17,125 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e4cdd3a0b883,41101,1731295033876, state=OPENING 2024-11-11T03:17:17,136 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-11T03:17:17,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:17:17,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:17:17,140 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T03:17:17,144 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T03:17:17,154 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=e4cdd3a0b883,41101,1731295033876}] 2024-11-11T03:17:17,353 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:17,355 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-11T03:17:17,382 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47900, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-11T03:17:17,403 INFO [RS_OPEN_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-11-11T03:17:17,404 INFO [RS_OPEN_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-11T03:17:17,411 INFO [RS_OPEN_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-11T03:17:17,425 INFO [RS_OPEN_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=e4cdd3a0b883%2C41101%2C1731295033876.meta, suffix=.meta, logDir=hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/WALs/e4cdd3a0b883,41101,1731295033876, archiveDir=hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/oldWALs, maxLogs=32 2024-11-11T03:17:17,462 DEBUG [RS_OPEN_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(600): When create output stream for /user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/WALs/e4cdd3a0b883,41101,1731295033876/e4cdd3a0b883%2C41101%2C1731295033876.meta.1731295037428.meta, exclude list is [], retry=0 2024-11-11T03:17:17,474 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43237,DS-d8921118-796e-46de-ab00-808a3cfc35de,DISK] 2024-11-11T03:17:17,507 INFO [RS_OPEN_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/WALs/e4cdd3a0b883,41101,1731295033876/e4cdd3a0b883%2C41101%2C1731295033876.meta.1731295037428.meta 2024-11-11T03:17:17,524 DEBUG [RS_OPEN_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:42839:42839)] 2024-11-11T03:17:17,524 DEBUG [RS_OPEN_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-11T03:17:17,527 DEBUG [RS_OPEN_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-11T03:17:17,635 DEBUG [RS_OPEN_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-11T03:17:17,643 INFO [RS_OPEN_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-11T03:17:17,653 DEBUG [RS_OPEN_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-11T03:17:17,653 DEBUG [RS_OPEN_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T03:17:17,654 DEBUG [RS_OPEN_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-11-11T03:17:17,654 DEBUG [RS_OPEN_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-11-11T03:17:17,669 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-11T03:17:17,673 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-11T03:17:17,673 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:17:17,674 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T03:17:17,675 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-11T03:17:17,679 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-11T03:17:17,680 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:17:17,681 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T03:17:17,682 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-11T03:17:17,685 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-11T03:17:17,685 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:17:17,687 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-11T03:17:17,689 DEBUG [RS_OPEN_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/data/hbase/meta/1588230740 2024-11-11T03:17:17,695 DEBUG [RS_OPEN_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/data/hbase/meta/1588230740 2024-11-11T03:17:17,699 DEBUG [RS_OPEN_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-11T03:17:17,703 DEBUG [RS_OPEN_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-11T03:17:17,705 INFO [RS_OPEN_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69860209, jitterRate=0.04099823534488678}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-11T03:17:17,707 DEBUG [RS_OPEN_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-11T03:17:17,733 INFO [RS_OPEN_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731295037343 2024-11-11T03:17:17,758 DEBUG [RS_OPEN_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-11T03:17:17,759 INFO [RS_OPEN_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-11-11T03:17:17,761 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:17,764 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as e4cdd3a0b883,41101,1731295033876, state=OPEN 2024-11-11T03:17:17,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T03:17:17,771 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-11T03:17:17,772 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T03:17:17,772 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-11T03:17:17,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-11-11T03:17:17,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=e4cdd3a0b883,41101,1731295033876 in 618 msec 2024-11-11T03:17:17,809 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-11-11T03:17:17,810 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 1.3810 sec 2024-11-11T03:17:17,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.9300 sec 2024-11-11T03:17:17,822 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731295037821, completionTime=-1 2024-11-11T03:17:17,822 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-11T03:17:17,822 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-11-11T03:17:17,906 DEBUG [hconnection-0x1a87b203-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:17:17,909 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47902, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:17:17,928 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-11-11T03:17:17,928 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731295097928 2024-11-11T03:17:17,928 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731295157928 2024-11-11T03:17:17,928 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 106 msec 2024-11-11T03:17:17,974 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e4cdd3a0b883,36699,1731295032496-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-11T03:17:17,975 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e4cdd3a0b883,36699,1731295032496-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T03:17:17,977 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e4cdd3a0b883,36699,1731295032496-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T03:17:17,980 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-e4cdd3a0b883:36699, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T03:17:17,984 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-11T03:17:17,993 DEBUG [master/e4cdd3a0b883:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-11-11T03:17:17,996 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-11-11T03:17:17,998 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-11T03:17:18,024 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-11-11T03:17:18,029 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-11-11T03:17:18,030 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:17:18,034 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-11T03:17:18,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43237 is added to blk_1073741835_1011 (size=358) 2024-11-11T03:17:18,097 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => cf6c7e153be0acd168cc278b105d726f, NAME => 'hbase:namespace,,1731295037996.cf6c7e153be0acd168cc278b105d726f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3 2024-11-11T03:17:18,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43237 is added to blk_1073741836_1012 (size=42) 2024-11-11T03:17:18,141 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1731295037996.cf6c7e153be0acd168cc278b105d726f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T03:17:18,142 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing cf6c7e153be0acd168cc278b105d726f, disabling compactions & flushes 2024-11-11T03:17:18,142 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1731295037996.cf6c7e153be0acd168cc278b105d726f. 2024-11-11T03:17:18,142 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1731295037996.cf6c7e153be0acd168cc278b105d726f. 2024-11-11T03:17:18,142 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1731295037996.cf6c7e153be0acd168cc278b105d726f. after waiting 0 ms 2024-11-11T03:17:18,142 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1731295037996.cf6c7e153be0acd168cc278b105d726f. 2024-11-11T03:17:18,142 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1731295037996.cf6c7e153be0acd168cc278b105d726f. 2024-11-11T03:17:18,142 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for cf6c7e153be0acd168cc278b105d726f: 2024-11-11T03:17:18,146 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-11-11T03:17:18,157 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1731295037996.cf6c7e153be0acd168cc278b105d726f.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731295038148"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731295038148"}]},"ts":"1731295038148"} 2024-11-11T03:17:18,195 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-11T03:17:18,199 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-11T03:17:18,203 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731295038199"}]},"ts":"1731295038199"} 2024-11-11T03:17:18,215 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-11-11T03:17:18,226 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=cf6c7e153be0acd168cc278b105d726f, ASSIGN}] 2024-11-11T03:17:18,232 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=cf6c7e153be0acd168cc278b105d726f, ASSIGN 2024-11-11T03:17:18,234 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=cf6c7e153be0acd168cc278b105d726f, ASSIGN; state=OFFLINE, location=e4cdd3a0b883,41101,1731295033876; forceNewPlan=false, retain=false 2024-11-11T03:17:18,385 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=cf6c7e153be0acd168cc278b105d726f, regionState=OPENING, regionLocation=e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:18,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure cf6c7e153be0acd168cc278b105d726f, server=e4cdd3a0b883,41101,1731295033876}] 2024-11-11T03:17:18,550 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:18,565 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1731295037996.cf6c7e153be0acd168cc278b105d726f. 2024-11-11T03:17:18,566 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => cf6c7e153be0acd168cc278b105d726f, NAME => 'hbase:namespace,,1731295037996.cf6c7e153be0acd168cc278b105d726f.', STARTKEY => '', ENDKEY => ''} 2024-11-11T03:17:18,567 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace cf6c7e153be0acd168cc278b105d726f 2024-11-11T03:17:18,567 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1731295037996.cf6c7e153be0acd168cc278b105d726f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-11T03:17:18,567 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for cf6c7e153be0acd168cc278b105d726f 2024-11-11T03:17:18,567 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for cf6c7e153be0acd168cc278b105d726f 2024-11-11T03:17:18,576 INFO [StoreOpener-cf6c7e153be0acd168cc278b105d726f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region cf6c7e153be0acd168cc278b105d726f 2024-11-11T03:17:18,580 INFO [StoreOpener-cf6c7e153be0acd168cc278b105d726f-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region cf6c7e153be0acd168cc278b105d726f columnFamilyName info 2024-11-11T03:17:18,580 DEBUG [StoreOpener-cf6c7e153be0acd168cc278b105d726f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-11T03:17:18,582 INFO [StoreOpener-cf6c7e153be0acd168cc278b105d726f-1 {}] regionserver.HStore(327): Store=cf6c7e153be0acd168cc278b105d726f/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-11T03:17:18,588 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/data/hbase/namespace/cf6c7e153be0acd168cc278b105d726f 2024-11-11T03:17:18,591 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/data/hbase/namespace/cf6c7e153be0acd168cc278b105d726f 2024-11-11T03:17:18,596 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for cf6c7e153be0acd168cc278b105d726f 2024-11-11T03:17:18,606 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/data/hbase/namespace/cf6c7e153be0acd168cc278b105d726f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-11T03:17:18,607 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened cf6c7e153be0acd168cc278b105d726f; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63367819, jitterRate=-0.055745914578437805}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-11T03:17:18,609 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for cf6c7e153be0acd168cc278b105d726f: 2024-11-11T03:17:18,614 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1731295037996.cf6c7e153be0acd168cc278b105d726f., pid=6, masterSystemTime=1731295038550 2024-11-11T03:17:18,620 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1731295037996.cf6c7e153be0acd168cc278b105d726f. 2024-11-11T03:17:18,620 INFO [RS_OPEN_PRIORITY_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1731295037996.cf6c7e153be0acd168cc278b105d726f. 2024-11-11T03:17:18,620 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=cf6c7e153be0acd168cc278b105d726f, regionState=OPEN, openSeqNum=2, regionLocation=e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:18,675 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-11-11T03:17:18,675 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure cf6c7e153be0acd168cc278b105d726f, server=e4cdd3a0b883,41101,1731295033876 in 235 msec 2024-11-11T03:17:18,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-11-11T03:17:18,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=cf6c7e153be0acd168cc278b105d726f, ASSIGN in 450 msec 2024-11-11T03:17:18,687 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-11T03:17:18,688 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731295038687"}]},"ts":"1731295038687"} 2024-11-11T03:17:18,692 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-11-11T03:17:18,698 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-11-11T03:17:18,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 685 msec 2024-11-11T03:17:18,729 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-11-11T03:17:18,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-11-11T03:17:18,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:17:18,740 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:17:18,789 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-11-11T03:17:18,818 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-11T03:17:18,829 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 49 msec 2024-11-11T03:17:18,841 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-11-11T03:17:18,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-11T03:17:18,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 32 msec 2024-11-11T03:17:18,891 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-11-11T03:17:18,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-11-11T03:17:18,895 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 4.773sec 2024-11-11T03:17:18,897 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-11T03:17:18,899 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-11T03:17:18,901 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-11T03:17:18,904 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-11T03:17:18,904 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-11T03:17:18,905 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e4cdd3a0b883,36699,1731295032496-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-11T03:17:18,906 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e4cdd3a0b883,36699,1731295032496-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-11T03:17:18,936 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-11-11T03:17:18,938 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-11T03:17:18,938 INFO [master/e4cdd3a0b883:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=e4cdd3a0b883,36699,1731295032496-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-11T03:17:19,009 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x665ee9b6 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@30bade74 2024-11-11T03:17:19,010 WARN [Time-limited test {}] client.ZKConnectionRegistry(90): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-11-11T03:17:19,058 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@618daf1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:19,061 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-11T03:17:19,061 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-11T03:17:19,077 DEBUG [hconnection-0x20fc3642-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-11T03:17:19,087 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47916, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-11T03:17:19,098 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=e4cdd3a0b883,36699,1731295032496 2024-11-11T03:17:19,101 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4b613db6 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@43143270 2024-11-11T03:17:19,121 DEBUG [ReadOnlyZKClient-127.0.0.1:54139@0x4b613db6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@251963ac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:19,151 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[0] Thread=222, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=261, ProcessCount=11, AvailableMemoryMB=7870 2024-11-11T03:17:19,200 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching master stub from registry 2024-11-11T03:17:19,202 DEBUG [ReadOnlyZKClient-127.0.0.1:54139@0x4b613db6 {}] client.AsyncConnectionImpl(310): The fetched master address is e4cdd3a0b883,36699,1731295032496 2024-11-11T03:17:19,206 DEBUG [ReadOnlyZKClient-127.0.0.1:54139@0x4b613db6 {}] client.ConnectionUtils(581): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5bef82fa 2024-11-11T03:17:19,211 DEBUG [ReadOnlyZKClient-127.0.0.1:54139@0x4b613db6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-11T03:17:19,223 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39256, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-11T03:17:19,225 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:17:19,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:17:19,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-11T03:17:19,271 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:19,272 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:19,276 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:19,288 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60004 connected 2024-11-11T03:17:19,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-11T03:17:19,394 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60004, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:19,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:17:19,407 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:17:19,425 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:19,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-11T03:17:19,591 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:19,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=10 2024-11-11T03:17:19,598 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-11T03:17:19,635 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=10}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:19,676 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:19,703 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-11T03:17:19,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=10 2024-11-11T03:17:19,713 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:19,718 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-11-11T03:17:19,718 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 288 msec 2024-11-11T03:17:19,718 INFO [PEWorker-5 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-11T03:17:19,722 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 491 msec 2024-11-11T03:17:19,816 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5a2078b7 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@46192707 2024-11-11T03:17:19,853 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:19,854 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:19,855 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@27fde1b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:19,858 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:19,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-11T03:17:19,878 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:17:19,882 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-11T03:17:19,887 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:19,888 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:19,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=11, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:19,899 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60006 connected 2024-11-11T03:17:19,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=11 2024-11-11T03:17:19,952 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:19,954 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:19,981 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:19,992 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60007 connected 2024-11-11T03:17:20,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=11 2024-11-11T03:17:20,096 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60007, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:20,102 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:20,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=11 2024-11-11T03:17:20,255 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:20,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=12 2024-11-11T03:17:20,257 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:20,259 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:20,259 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:20,259 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:20,260 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60006, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:20,260 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60006, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:20,363 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60006, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:20,364 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=12}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5a2078b7 to 127.0.0.1:54139 2024-11-11T03:17:20,364 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=12}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:20,392 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:20,393 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:20,394 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-11T03:17:20,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=12 2024-11-11T03:17:20,399 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:20,404 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=12, resume processing ppid=11 2024-11-11T03:17:20,404 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, ppid=11, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 298 msec 2024-11-11T03:17:20,404 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-11T03:17:20,417 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 522 msec 2024-11-11T03:17:20,468 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4ec623f0 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@475fb0e1 2024-11-11T03:17:20,493 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:20,494 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:20,495 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@63a5a877, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:20,497 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:20,524 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:20,525 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60009 connected 2024-11-11T03:17:20,527 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:20,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=11 2024-11-11T03:17:20,548 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:20,553 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:20,555 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:20,556 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:20,559 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-11-11T03:17:20,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=13, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:20,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=13 2024-11-11T03:17:20,588 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:20,590 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:20,593 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:20,594 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6000a connected 2024-11-11T03:17:20,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=13 2024-11-11T03:17:20,696 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6000a, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:20,707 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:20,862 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:20,864 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=14 2024-11-11T03:17:20,864 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:20,866 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:20,866 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:20,866 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:20,867 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60009, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:20,867 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60009, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:20,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=13 2024-11-11T03:17:20,976 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60009, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:20,976 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=14}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4ec623f0 to 127.0.0.1:54139 2024-11-11T03:17:20,976 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=14}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:21,007 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:21,008 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:21,013 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-11T03:17:21,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=14 2024-11-11T03:17:21,022 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:21,026 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=14, resume processing ppid=13 2024-11-11T03:17:21,027 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, ppid=13, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 315 msec 2024-11-11T03:17:21,027 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-11-11T03:17:21,033 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 468 msec 2024-11-11T03:17:21,064 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x77ff5040 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@42db8af5 2024-11-11T03:17:21,077 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:21,077 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:21,078 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@4a21b661, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:21,080 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:21,091 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:21,092 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:21,093 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e6000c connected 2024-11-11T03:17:21,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=13 2024-11-11T03:17:21,197 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:21,198 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:21,200 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:21,201 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:21,203 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:21,204 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-11-11T03:17:21,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=15, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:21,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-11T03:17:21,233 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:21,235 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:21,238 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:21,239 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6000d connected 2024-11-11T03:17:21,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-11T03:17:21,341 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6000d, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:21,348 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:21,503 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:21,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=16 2024-11-11T03:17:21,504 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:21,506 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:21,506 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:21,506 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:21,506 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e6000c, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:21,507 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e6000c, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:21,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-11T03:17:21,614 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e6000c, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:21,614 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=16}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x77ff5040 to 127.0.0.1:54139 2024-11-11T03:17:21,614 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=16}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:21,640 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:21,641 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:21,644 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-11T03:17:21,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=16 2024-11-11T03:17:21,650 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:21,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=16, resume processing ppid=15 2024-11-11T03:17:21,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, ppid=15, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 302 msec 2024-11-11T03:17:21,655 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-11-11T03:17:21,658 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 451 msec 2024-11-11T03:17:21,690 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4d9fe477 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@28392aac 2024-11-11T03:17:21,700 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:21,700 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:21,700 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@463f07d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:21,702 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:21,706 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:21,707 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:21,707 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e6000f connected 2024-11-11T03:17:21,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-11T03:17:21,837 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:21,838 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:21,840 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:21,841 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:21,843 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-11T03:17:21,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=17, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:21,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=17 2024-11-11T03:17:21,877 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:21,878 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:21,883 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:21,884 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60010 connected 2024-11-11T03:17:21,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=17 2024-11-11T03:17:21,987 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60010, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:21,995 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:22,150 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:22,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=18 2024-11-11T03:17:22,151 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:22,155 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:22,155 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:22,155 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:22,155 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e6000f, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:22,156 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e6000f, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=17 2024-11-11T03:17:22,259 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e6000f, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:22,259 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=18}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4d9fe477 to 127.0.0.1:54139 2024-11-11T03:17:22,259 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=18}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:22,283 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:22,284 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:22,284 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-11T03:17:22,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=18 2024-11-11T03:17:22,289 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:22,296 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=18, resume processing ppid=17 2024-11-11T03:17:22,296 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, ppid=17, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 294 msec 2024-11-11T03:17:22,297 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-11T03:17:22,299 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 454 msec 2024-11-11T03:17:22,329 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x17ab34b6 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@797cdc69 2024-11-11T03:17:22,333 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:22,334 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:22,334 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@165e2abf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:22,335 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:22,338 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:22,340 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:22,344 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60012 connected 2024-11-11T03:17:22,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=17 2024-11-11T03:17:22,477 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:22,477 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:22,479 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:22,481 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-11-11T03:17:22,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=19, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:22,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=19 2024-11-11T03:17:22,497 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-11T03:17:22,498 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-11-11T03:17:22,510 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:22,511 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:22,514 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:22,515 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60013 connected 2024-11-11T03:17:22,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=19 2024-11-11T03:17:22,619 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60013, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:22,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:22,783 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:22,785 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=20 2024-11-11T03:17:22,785 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:22,787 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:22,787 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:22,787 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:22,787 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60012, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:22,788 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60012, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:22,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=19 2024-11-11T03:17:22,891 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60012, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:22,891 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=20}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x17ab34b6 to 127.0.0.1:54139 2024-11-11T03:17:22,891 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=20}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:22,918 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:22,918 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:22,919 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-11T03:17:22,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=20 2024-11-11T03:17:22,926 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:22,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=20, resume processing ppid=19 2024-11-11T03:17:22,930 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-11-11T03:17:22,930 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=20, ppid=19, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 300 msec 2024-11-11T03:17:22,935 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=19, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 449 msec 2024-11-11T03:17:22,963 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x68690d64 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@28430bd8 2024-11-11T03:17:22,970 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:22,971 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:22,971 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@72d7f480, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:22,973 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:22,976 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:22,976 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:22,980 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60015 connected 2024-11-11T03:17:23,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=19 2024-11-11T03:17:23,118 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:23,118 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:23,120 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-11T03:17:23,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=21, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:23,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-11T03:17:23,169 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:23,169 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T03:17:23,188 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:23,209 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:23,219 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60016 connected 2024-11-11T03:17:23,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-11T03:17:23,310 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-11T03:17:23,310 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-11T03:17:23,313 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-11-11T03:17:23,314 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2024-11-11T03:17:23,316 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T03:17:23,316 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-11T03:17:23,316 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-11T03:17:23,316 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-11T03:17:23,323 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60016, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:23,330 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:23,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-11T03:17:23,484 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:23,485 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=22 2024-11-11T03:17:23,485 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:23,487 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:23,487 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:23,488 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:23,488 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60015, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:23,488 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60015, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:23,592 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60015, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:23,592 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=22}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x68690d64 to 127.0.0.1:54139 2024-11-11T03:17:23,592 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=22}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:23,611 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:23,611 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:23,611 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-11T03:17:23,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=22 2024-11-11T03:17:23,617 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:23,621 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=22, resume processing ppid=21 2024-11-11T03:17:23,621 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-11T03:17:23,621 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=22, ppid=21, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 288 msec 2024-11-11T03:17:23,624 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=21, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 501 msec 2024-11-11T03:17:23,655 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x24b31498 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5160ce8c 2024-11-11T03:17:23,689 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:23,689 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:23,690 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@2d78a8db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:23,692 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:23,703 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:23,703 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:23,707 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60018 connected 2024-11-11T03:17:23,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-11T03:17:23,756 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:23,757 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:23,762 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:17:23,762 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:17:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=24, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:23,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=23, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:23,772 DEBUG [PEWorker-2 {}] procedure.MasterProcedureScheduler(583): PEER '1', shared lock count=0, exclusively locked by procId=23 2024-11-11T03:17:23,772 DEBUG [PEWorker-2 {}] procedure2.ProcedureExecutor(1470): LOCK_EVENT_WAIT pid=24, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:23,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-11T03:17:23,779 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-11T03:17:23,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-11T03:17:23,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-11T03:17:23,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:17:23,787 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:17:23,789 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=25, ppid=23, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-11T03:17:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-11T03:17:23,943 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:23,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=25 2024-11-11T03:17:23,945 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-11T03:17:23,945 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-11T03:17:23,945 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-11T03:17:23,945 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:23,946 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60018, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:23,946 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60018, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:24,052 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60018, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:24,052 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=25}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x24b31498 to 127.0.0.1:54139 2024-11-11T03:17:24,052 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=25}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:24,053 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-11T03:17:24,056 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=25}] zookeeper.ZKUtil(111): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:17:24,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:17:24,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:17:24,060 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:17:24,062 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=25}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:17:24,062 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=25 2024-11-11T03:17:24,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=25 2024-11-11T03:17:24,073 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:24,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=25, resume processing ppid=23 2024-11-11T03:17:24,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=25, ppid=23, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 284 msec 2024-11-11T03:17:24,081 DEBUG [PEWorker-3 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:17:24,082 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-11T03:17:24,085 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=23, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 320 msec 2024-11-11T03:17:24,086 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:24,095 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=24, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:24,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-11T03:17:24,097 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:17:24,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-11T03:17:24,098 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:17:24,098 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=24, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=332 msec 2024-11-11T03:17:24,099 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:24,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=26, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:24,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-11-11T03:17:24,103 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:24,105 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=26, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:24,107 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=26, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=7 msec 2024-11-11T03:17:24,107 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:24,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-11-11T03:17:24,208 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:17:24,209 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=testing utility connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:24,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility0x0, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:24,215 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): testing utility-0x101955526e60019 connected 2024-11-11T03:17:24,246 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[0] Thread=223 (was 222) Potentially hanging thread: RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) - Thread LEAK? -, OpenFileDescriptor=452 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=352 (was 261) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7817 (was 7870) 2024-11-11T03:17:24,269 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[0] Thread=223, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=352, ProcessCount=11, AvailableMemoryMB=7817 2024-11-11T03:17:24,271 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:17:24,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=27, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:17:24,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-11T03:17:24,300 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:24,302 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:24,305 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:24,312 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6001a connected 2024-11-11T03:17:24,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-11T03:17:24,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-11T03:17:24,408 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-11T03:17:24,415 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6001a, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:24,428 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:24,583 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:24,585 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=28 2024-11-11T03:17:24,585 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-11T03:17:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-11T03:17:24,612 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=28}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:24,641 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:24,675 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-11T03:17:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=28 2024-11-11T03:17:24,692 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:24,698 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=28, resume processing ppid=27 2024-11-11T03:17:24,699 INFO [PEWorker-2 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-11T03:17:24,699 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=28, ppid=27, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 264 msec 2024-11-11T03:17:24,703 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=27, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 428 msec 2024-11-11T03:17:24,724 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3cf8c953 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@e699ebb 2024-11-11T03:17:24,750 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:24,750 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:24,751 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@636c4800, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:24,752 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:24,756 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:24,757 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:24,757 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e6001c connected 2024-11-11T03:17:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-11T03:17:24,907 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:17:24,908 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-11T03:17:24,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=29, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:24,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=29 2024-11-11T03:17:24,934 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:24,935 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:24,938 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:25,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=29 2024-11-11T03:17:25,041 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:25,048 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:25,202 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:25,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=30 2024-11-11T03:17:25,203 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:25,207 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:25,207 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:25,207 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:25,208 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e6001c, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:25,208 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e6001c, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:25,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=29 2024-11-11T03:17:25,310 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e6001c, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:25,310 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=30}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3cf8c953 to 127.0.0.1:54139 2024-11-11T03:17:25,310 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=30}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:25,335 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:25,335 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:25,336 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-11-11T03:17:25,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=30 2024-11-11T03:17:25,341 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:25,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=30, resume processing ppid=29 2024-11-11T03:17:25,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=30, ppid=29, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 293 msec 2024-11-11T03:17:25,345 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-11T03:17:25,348 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=29, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 437 msec 2024-11-11T03:17:25,377 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x60114f58 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4a1facc7 2024-11-11T03:17:25,381 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:25,382 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:25,382 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3a94258d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:25,384 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:25,387 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:25,387 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:25,387 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e6001f connected 2024-11-11T03:17:25,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=29 2024-11-11T03:17:25,536 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:25,537 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:25,539 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-11-11T03:17:25,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=31, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:25,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=31 2024-11-11T03:17:25,564 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:25,565 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:25,569 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:25,570 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60020 connected 2024-11-11T03:17:25,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=31 2024-11-11T03:17:25,678 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60020, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:25,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:25,837 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:25,838 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=32 2024-11-11T03:17:25,839 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:25,840 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:25,840 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:25,840 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:25,841 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e6001f, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:25,841 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e6001f, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:25,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=31 2024-11-11T03:17:25,943 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e6001f, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:25,943 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=32}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x60114f58 to 127.0.0.1:54139 2024-11-11T03:17:25,944 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=32}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:25,968 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:25,968 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:25,972 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=32 2024-11-11T03:17:25,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=32 2024-11-11T03:17:25,977 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:25,980 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=32, resume processing ppid=31 2024-11-11T03:17:25,980 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-11-11T03:17:25,980 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=32, ppid=31, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 292 msec 2024-11-11T03:17:25,983 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=31, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 441 msec 2024-11-11T03:17:26,019 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x13b7c8d0 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7e0e91cd 2024-11-11T03:17:26,035 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:26,036 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:26,036 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@13473c93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:26,038 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:26,044 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:26,045 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:26,046 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60022 connected 2024-11-11T03:17:26,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=31 2024-11-11T03:17:26,167 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:26,168 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:26,170 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns1:testNamespacesAndTableCfsConfigConflict1=null},bandwidth=0,serial=false 2024-11-11T03:17:26,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=33, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:26,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-11T03:17:26,195 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:26,196 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:26,201 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:26,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-11T03:17:26,312 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:26,313 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:26,315 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=33, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:26,317 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=33, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=145 msec 2024-11-11T03:17:26,317 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:26,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-11T03:17:26,498 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config 2024-11-11T03:17:26,498 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:26,500 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-11-11T03:17:26,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=34, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-11-11T03:17:26,529 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:26,530 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:26,533 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:26,536 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60024 connected 2024-11-11T03:17:26,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-11-11T03:17:26,639 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60024, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:26,647 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:26,800 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:26,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=35 2024-11-11T03:17:26,802 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:26,803 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:26,803 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:26,804 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:26,804 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60022, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:26,804 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60022, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:26,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-11-11T03:17:26,906 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60022, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:26,907 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=35}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x13b7c8d0 to 127.0.0.1:54139 2024-11-11T03:17:26,907 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=35}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:26,926 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:26,926 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:26,926 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-11T03:17:26,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=35 2024-11-11T03:17:26,930 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:26,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=35, resume processing ppid=34 2024-11-11T03:17:26,935 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=35, ppid=34, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 284 msec 2024-11-11T03:17:26,935 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-11-11T03:17:26,937 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 435 msec 2024-11-11T03:17:26,963 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2319840c to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4cbfc2e9 2024-11-11T03:17:26,984 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:26,985 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:26,985 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@65cca43f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:26,986 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:26,990 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:26,991 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:26,991 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60026 connected 2024-11-11T03:17:27,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-11-11T03:17:27,127 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:27,127 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:27,129 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-11-11T03:17:27,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=36, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:27,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-11T03:17:27,155 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:27,156 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:27,159 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:27,160 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60027 connected 2024-11-11T03:17:27,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-11T03:17:27,262 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60027, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:27,262 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:27,265 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=36, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:27,267 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=36, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=136 msec 2024-11-11T03:17:27,267 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:27,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-11T03:17:27,447 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config 2024-11-11T03:17:27,448 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:17:27,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=37, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:27,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=37 2024-11-11T03:17:27,454 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-11T03:17:27,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-11T03:17:27,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:17:27,457 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:17:27,458 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:27,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=37 2024-11-11T03:17:27,610 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:27,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=38 2024-11-11T03:17:27,612 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-11T03:17:27,612 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-11T03:17:27,612 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-11T03:17:27,612 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:27,613 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60026, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:27,613 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60026, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:27,718 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60026, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:27,718 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=38}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2319840c to 127.0.0.1:54139 2024-11-11T03:17:27,718 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=38}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:27,719 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-11T03:17:27,726 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=38}] zookeeper.ZKUtil(111): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:17:27,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:17:27,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:17:27,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:17:27,731 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=38}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:17:27,732 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=38 2024-11-11T03:17:27,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=38 2024-11-11T03:17:27,737 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:27,740 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=38, resume processing ppid=37 2024-11-11T03:17:27,741 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=38, ppid=37, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 279 msec 2024-11-11T03:17:27,745 DEBUG [PEWorker-3 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:17:27,745 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-11T03:17:27,748 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=37, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 297 msec 2024-11-11T03:17:27,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=37 2024-11-11T03:17:27,767 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:17:27,768 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:17:27,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=39, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:27,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-11T03:17:27,771 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:27,772 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=39, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:27,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=39, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-11-11T03:17:27,774 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:27,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-11T03:17:27,877 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-11T03:17:27,878 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:17:27,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=40, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:27,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=40 2024-11-11T03:17:27,881 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:27,883 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=40, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:27,885 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=40, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-11-11T03:17:27,885 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:27,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=40 2024-11-11T03:17:27,987 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:17:28,008 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[0] Thread=223 (was 223), OpenFileDescriptor=452 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=364 (was 352) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7795 (was 7817) 2024-11-11T03:17:28,021 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAddRemovePeer[0] Thread=223, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=364, ProcessCount=11, AvailableMemoryMB=7794 2024-11-11T03:17:28,022 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:17:28,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=41, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:17:28,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41 2024-11-11T03:17:28,047 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:28,048 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:28,051 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:28,052 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60028 connected 2024-11-11T03:17:28,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41 2024-11-11T03:17:28,154 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60028, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:28,163 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:28,317 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:28,318 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=42 2024-11-11T03:17:28,319 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=42}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-11T03:17:28,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41 2024-11-11T03:17:28,347 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=42}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:28,365 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=42}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:28,376 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=42}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=42 2024-11-11T03:17:28,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=42 2024-11-11T03:17:28,380 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:28,383 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=42, resume processing ppid=41 2024-11-11T03:17:28,383 INFO [PEWorker-1 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-11T03:17:28,383 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=42, ppid=41, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 217 msec 2024-11-11T03:17:28,385 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=41, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 361 msec 2024-11-11T03:17:28,421 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x049859a3 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2342156d 2024-11-11T03:17:28,429 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:28,429 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:28,429 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@38a7923c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:28,431 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:28,436 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:28,437 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:28,440 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e6002a connected 2024-11-11T03:17:28,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41 2024-11-11T03:17:28,657 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:17:28,658 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:17:28,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=43, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:17:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=43 2024-11-11T03:17:28,691 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:28,692 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:28,713 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:28,714 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6002b connected 2024-11-11T03:17:28,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=43 2024-11-11T03:17:28,816 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6002b, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:28,816 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:121) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:28,818 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=43, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:121) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:28,822 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=43, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=161 msec 2024-11-11T03:17:28,822 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:28,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=43 2024-11-11T03:17:28,987 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Replication peer 1 already exists 2024-11-11T03:17:28,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-11-11T03:17:28,996 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:17:28,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=44, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:28,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=44 2024-11-11T03:17:29,002 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:29,004 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=44, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:29,006 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=44, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=8 msec 2024-11-11T03:17:29,006 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:29,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=44 2024-11-11T03:17:29,108 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:17:29,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-11-11T03:17:29,110 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=2, config=clusterKey=127.0.0.1:54139:/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:17:29,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=45, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:17:29,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-11T03:17:29,133 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:29,134 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:29,137 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:29,138 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6002c connected 2024-11-11T03:17:29,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-11T03:17:29,240 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6002c, quorum=127.0.0.1:54139, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:29,253 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:29,406 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:29,407 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=46 2024-11-11T03:17:29,407 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=2, type=ADD_PEER 2024-11-11T03:17:29,426 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=46}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:29,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-11T03:17:29,438 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 2000 millis 2024-11-11T03:17:29,445 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.ReplicationSource(229): queueId=2, ReplicationSource: 2, currentBandwidth=0 2024-11-11T03:17:29,451 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=46 2024-11-11T03:17:29,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=46 2024-11-11T03:17:29,455 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 2 for ADD on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:29,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=46, resume processing ppid=45 2024-11-11T03:17:29,458 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=46, ppid=45, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 202 msec 2024-11-11T03:17:29,458 INFO [PEWorker-2 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 2, config clusterKey=127.0.0.1:54139:/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-11T03:17:29,475 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=45, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 348 msec 2024-11-11T03:17:29,493 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6dfcb929 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3ce17511 2024-11-11T03:17:29,521 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,2 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:29,522 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,2 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:29,522 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@153ec4be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:29,524 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 2 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:29,527 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,2-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 20x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:29,527 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,2 {}] regionserver.ReplicationSource(569): peerId=2, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:29,532 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 2-0x101955526e6002e connected 2024-11-11T03:17:29,555 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T03:17:29,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-11T03:17:29,737 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 2 completed 2024-11-11T03:17:29,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-11-11T03:17:29,739 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:17:29,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=47, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:29,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-11-11T03:17:29,744 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-11T03:17:29,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-11T03:17:29,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:17:29,747 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:17:29,748 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:29,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-11-11T03:17:29,905 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:29,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=48 2024-11-11T03:17:29,907 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-11T03:17:29,907 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-11T03:17:29,907 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-11T03:17:29,907 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:29,907 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e6002a, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:29,908 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e6002a, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:30,013 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e6002a, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:30,013 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=48}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x049859a3 to 127.0.0.1:54139 2024-11-11T03:17:30,014 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=48}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:30,014 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-11T03:17:30,018 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=48}] zookeeper.ZKUtil(111): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:17:30,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:17:30,022 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:17:30,023 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:17:30,027 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=48}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:17:30,028 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=48 2024-11-11T03:17:30,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=48 2024-11-11T03:17:30,032 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:30,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=48, resume processing ppid=47 2024-11-11T03:17:30,041 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=48, ppid=47, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 284 msec 2024-11-11T03:17:30,047 DEBUG [PEWorker-1 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:17:30,047 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-11T03:17:30,052 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=47, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 308 msec 2024-11-11T03:17:30,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-11-11T03:17:30,057 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:17:30,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-11-11T03:17:30,059 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:17:30,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=49, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:30,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-11-11T03:17:30,064 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/peer-state 2024-11-11T03:17:30,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/peer-state 2024-11-11T03:17:30,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/2 2024-11-11T03:17:30,068 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2 2024-11-11T03:17:30,070 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:30,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-11-11T03:17:30,222 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:30,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=50 2024-11-11T03:17:30,223 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=2, type=REMOVE_PEER 2024-11-11T03:17:30,223 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 2: 0 2024-11-11T03:17:30,224 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.ReplicationSource(657): peerId=2, Closing source 2 because: Replication stream was removed by a user 2024-11-11T03:17:30,224 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,2 {}] regionserver.ReplicationSource(508): peerId=2, Interrupted while sleeping between retries 2024-11-11T03:17:30,224 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,2 {}] zookeeper.ZKUtil(165): connection to cluster: 2-0x101955526e6002e, quorum=127.0.0.1:54139, baseZNode=/hbase-test2 Unable to set watcher on znode (/hbase-test2/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:30,224 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,2 {}] zookeeper.ZKWatcher(734): connection to cluster: 2-0x101955526e6002e, quorum=127.0.0.1:54139, baseZNode=/hbase-test2 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:30,326 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,2-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 2-0x101955526e6002e, quorum=127.0.0.1:54139, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:30,326 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=50}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6dfcb929 to 127.0.0.1:54139 2024-11-11T03:17:30,326 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=50}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:30,327 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.ReplicationSourceManager(430): Done with the queue 2 2024-11-11T03:17:30,329 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=50}] zookeeper.ZKUtil(111): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/2/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:17:30,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/2/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:17:30,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/2 2024-11-11T03:17:30,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/2 2024-11-11T03:17:30,333 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=50}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/2 not found in hfile reference queue. 2024-11-11T03:17:30,333 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=50 2024-11-11T03:17:30,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=50 2024-11-11T03:17:30,336 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 2 for REMOVE on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:30,339 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=50, resume processing ppid=49 2024-11-11T03:17:30,339 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=50, ppid=49, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 267 msec 2024-11-11T03:17:30,343 DEBUG [PEWorker-3 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/2 not found in hfile reference queue. 2024-11-11T03:17:30,343 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(76): Successfully removed peer 2 2024-11-11T03:17:30,345 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=49, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 284 msec 2024-11-11T03:17:30,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-11-11T03:17:30,376 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 2 completed 2024-11-11T03:17:30,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-11-11T03:17:30,378 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:17:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=51, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:30,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-11T03:17:30,381 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:30,382 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=51, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:30,383 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=51, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-11-11T03:17:30,383 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:30,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-11T03:17:30,487 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-11T03:17:30,487 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:17:30,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=52, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:30,491 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:30,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=52 2024-11-11T03:17:30,493 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=52, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:30,495 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=52, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=7 msec 2024-11-11T03:17:30,495 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:30,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=52 2024-11-11T03:17:30,598 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:17:30,620 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAddRemovePeer[0] Thread=220 (was 223), OpenFileDescriptor=444 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=383 (was 364) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7760 (was 7794) 2024-11-11T03:17:30,633 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[0] Thread=220, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=383, ProcessCount=11, AvailableMemoryMB=7758 2024-11-11T03:17:30,634 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=whatever,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:17:30,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=53, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:17:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53 2024-11-11T03:17:30,638 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:291) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:288) ~[classes/:?] ... 12 more 2024-11-11T03:17:30,639 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=53, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:291) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:288) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:30,641 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=53, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=6 msec 2024-11-11T03:17:30,642 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:30,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53 2024-11-11T03:17:30,748 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Can not instantiate configured replication endpoint class=whatever 2024-11-11T03:17:30,753 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:17:30,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=54, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:30,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-11T03:17:30,757 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:30,758 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=54, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:30,760 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=54, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-11-11T03:17:30,760 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:30,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-11T03:17:30,872 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-11T03:17:30,873 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:17:30,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=55, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:30,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-11-11T03:17:30,876 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:30,878 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=55, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:30,880 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=55, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-11-11T03:17:30,880 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-11-11T03:17:30,987 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:17:31,005 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[0] Thread=220 (was 220), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=383 (was 383), ProcessCount=11 (was 11), AvailableMemoryMB=7753 (was 7758) 2024-11-11T03:17:31,017 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[0] Thread=220, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=383, ProcessCount=11, AvailableMemoryMB=7752 2024-11-11T03:17:31,019 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=whatever,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:17:31,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=56, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:17:31,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=56 2024-11-11T03:17:31,024 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] ... 13 more 2024-11-11T03:17:31,025 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=56, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:31,034 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=56, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=5 msec 2024-11-11T03:17:31,034 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:31,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=56 2024-11-11T03:17:31,136 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Invalid cluster key: whatever 2024-11-11T03:17:31,137 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:17:31,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=57, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:31,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-11T03:17:31,140 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:31,141 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=57, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:31,142 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=57, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-11-11T03:17:31,142 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:31,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-11T03:17:31,247 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-11T03:17:31,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:17:31,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=58, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:31,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=58 2024-11-11T03:17:31,250 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:31,252 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=58, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:31,253 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=58, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-11-11T03:17:31,253 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:31,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=58 2024-11-11T03:17:31,357 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:17:31,373 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[0] Thread=220 (was 220), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=383 (was 383), ProcessCount=11 (was 11), AvailableMemoryMB=7748 (was 7752) 2024-11-11T03:17:31,385 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[0] Thread=220, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=383, ProcessCount=11, AvailableMemoryMB=7748 2024-11-11T03:17:31,386 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:17:31,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=59, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:17:31,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-11-11T03:17:31,409 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:31,410 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:31,412 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:31,413 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6002f connected 2024-11-11T03:17:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-11-11T03:17:31,515 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6002f, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:31,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:31,675 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:31,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=60 2024-11-11T03:17:31,676 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-11T03:17:31,701 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=60}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:31,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-11-11T03:17:31,718 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:31,726 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=60 2024-11-11T03:17:31,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=60 2024-11-11T03:17:31,731 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:31,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=60, resume processing ppid=59 2024-11-11T03:17:31,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=60, ppid=59, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 209 msec 2024-11-11T03:17:31,735 INFO [PEWorker-2 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-11T03:17:31,737 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=59, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 348 msec 2024-11-11T03:17:31,771 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x07606fa3 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6a58dee6 2024-11-11T03:17:31,775 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:31,775 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:31,776 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@18fd5b80, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:31,777 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:31,781 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:31,781 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:31,782 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60031 connected 2024-11-11T03:17:32,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-11-11T03:17:32,017 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:17:32,018 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-11T03:17:32,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=61, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:32,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-11-11T03:17:32,042 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:32,042 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:32,045 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:32,048 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60032 connected 2024-11-11T03:17:32,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-11-11T03:17:32,150 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60032, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:32,161 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:32,314 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:32,315 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=62 2024-11-11T03:17:32,315 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:32,317 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:32,317 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:32,317 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:32,317 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60031, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:32,317 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60031, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:32,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-11-11T03:17:32,421 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60031, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:32,421 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=62}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x07606fa3 to 127.0.0.1:54139 2024-11-11T03:17:32,421 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=62}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:32,443 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:32,443 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:32,444 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-11-11T03:17:32,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=62 2024-11-11T03:17:32,447 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:32,450 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=62, resume processing ppid=61 2024-11-11T03:17:32,450 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=62, ppid=61, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 286 msec 2024-11-11T03:17:32,450 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-11T03:17:32,452 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=61, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 432 msec 2024-11-11T03:17:32,484 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6dcce862 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@565222f0 2024-11-11T03:17:32,488 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:32,489 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:32,489 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@1ab6525c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:32,491 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:32,498 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:32,498 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:32,498 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60034 connected 2024-11-11T03:17:32,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-11-11T03:17:32,647 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:32,647 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:32,648 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-11-11T03:17:32,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=63, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:32,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-11T03:17:32,666 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:32,667 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:32,669 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:32,670 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60035 connected 2024-11-11T03:17:32,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-11T03:17:32,772 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60035, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:32,776 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:32,928 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:32,929 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=64 2024-11-11T03:17:32,929 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:32,930 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:32,930 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:32,930 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:32,931 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60034, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:32,931 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60034, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:32,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-11T03:17:33,033 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60034, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:33,033 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=64}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6dcce862 to 127.0.0.1:54139 2024-11-11T03:17:33,033 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=64}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:33,049 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:33,050 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:33,050 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=64 2024-11-11T03:17:33,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=64 2024-11-11T03:17:33,053 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:33,055 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=64, resume processing ppid=63 2024-11-11T03:17:33,055 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=64, ppid=63, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 277 msec 2024-11-11T03:17:33,055 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-11-11T03:17:33,057 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=63, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 407 msec 2024-11-11T03:17:33,081 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x354851f8 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6cba3984 2024-11-11T03:17:33,085 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:33,085 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:33,086 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3b2e5300, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:33,087 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:33,090 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:33,091 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:33,091 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60037 connected 2024-11-11T03:17:33,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-11T03:17:33,276 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:33,277 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:33,278 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:33,278 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-11-11T03:17:33,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=65, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:33,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-11-11T03:17:33,296 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:33,296 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:33,299 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:33,299 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60038 connected 2024-11-11T03:17:33,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-11-11T03:17:33,401 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60038, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:33,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:33,557 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:33,558 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=66 2024-11-11T03:17:33,558 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:33,559 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:33,560 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:33,560 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:33,560 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60037, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:33,560 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60037, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:33,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-11-11T03:17:33,662 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60037, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:33,662 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=66}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x354851f8 to 127.0.0.1:54139 2024-11-11T03:17:33,663 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=66}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:33,678 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:33,678 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:33,679 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-11-11T03:17:33,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=66 2024-11-11T03:17:33,681 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:33,683 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=66, resume processing ppid=65 2024-11-11T03:17:33,683 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-11-11T03:17:33,684 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=66, ppid=65, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 276 msec 2024-11-11T03:17:33,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=65, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 406 msec 2024-11-11T03:17:33,709 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4276af3f to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@545da869 2024-11-11T03:17:33,713 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:33,713 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:33,713 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@690ea369, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:33,715 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:33,717 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:33,718 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e6003a connected 2024-11-11T03:17:33,718 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:33,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-11-11T03:17:33,906 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:33,907 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:33,908 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:17:33,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=67, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:33,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67 2024-11-11T03:17:33,912 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-11T03:17:33,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-11T03:17:33,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:17:33,915 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:17:33,916 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:34,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67 2024-11-11T03:17:34,068 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:34,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=68 2024-11-11T03:17:34,069 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-11T03:17:34,070 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-11T03:17:34,070 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-11T03:17:34,070 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:34,070 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e6003a, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:34,070 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e6003a, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:34,175 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e6003a, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:34,175 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=68}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4276af3f to 127.0.0.1:54139 2024-11-11T03:17:34,176 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=68}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:34,176 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-11T03:17:34,178 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=68}] zookeeper.ZKUtil(111): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:17:34,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:17:34,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:17:34,180 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:17:34,181 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=68}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:17:34,181 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=68 2024-11-11T03:17:34,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=68 2024-11-11T03:17:34,184 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:34,187 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=68, resume processing ppid=67 2024-11-11T03:17:34,187 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=68, ppid=67, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 268 msec 2024-11-11T03:17:34,190 DEBUG [PEWorker-4 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:17:34,190 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-11T03:17:34,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=67, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 282 msec 2024-11-11T03:17:34,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67 2024-11-11T03:17:34,227 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:17:34,228 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:17:34,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=69, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:34,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-11T03:17:34,230 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:34,231 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=69, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:34,232 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=69, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-11-11T03:17:34,232 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:34,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-11T03:17:34,337 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-11T03:17:34,338 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:17:34,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=70, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:34,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=70 2024-11-11T03:17:34,340 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:34,341 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=70, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:34,342 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=70, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-11-11T03:17:34,342 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:34,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=70 2024-11-11T03:17:34,447 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:17:34,459 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[0] Thread=220 (was 220), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=383 (was 383), ProcessCount=11 (was 11), AvailableMemoryMB=7723 (was 7748) 2024-11-11T03:17:34,467 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[0] Thread=220, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=383, ProcessCount=11, AvailableMemoryMB=7723 2024-11-11T03:17:34,468 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:17:34,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=71, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:17:34,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-11-11T03:17:34,486 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:34,487 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:34,490 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:34,490 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6003b connected 2024-11-11T03:17:34,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-11-11T03:17:34,592 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6003b, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:34,599 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:34,750 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:34,751 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=72 2024-11-11T03:17:34,751 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-11T03:17:34,768 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=72}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:34,783 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:34,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-11-11T03:17:34,792 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=72 2024-11-11T03:17:34,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=72 2024-11-11T03:17:34,795 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:34,797 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=72, resume processing ppid=71 2024-11-11T03:17:34,797 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=72, ppid=71, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 197 msec 2024-11-11T03:17:34,797 INFO [PEWorker-5 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-11T03:17:34,798 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=71, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 329 msec 2024-11-11T03:17:34,826 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x77257be9 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1ea791e 2024-11-11T03:17:34,833 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:34,834 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:34,834 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@76bddea3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:34,835 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:34,841 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:34,841 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:34,842 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e6003d connected 2024-11-11T03:17:35,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-11-11T03:17:35,097 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:17:35,097 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-11T03:17:35,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=73, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:35,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-11-11T03:17:35,115 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:35,115 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:35,121 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:35,122 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6003e connected 2024-11-11T03:17:35,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-11-11T03:17:35,223 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6003e, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:35,227 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:35,379 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:35,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=74 2024-11-11T03:17:35,380 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:35,381 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:35,381 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:35,381 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:35,382 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e6003d, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:35,382 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e6003d, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:35,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-11-11T03:17:35,484 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e6003d, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:35,484 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=74}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x77257be9 to 127.0.0.1:54139 2024-11-11T03:17:35,484 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=74}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:35,500 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:35,500 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:35,500 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=74 2024-11-11T03:17:35,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=74 2024-11-11T03:17:35,503 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:35,505 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=74, resume processing ppid=73 2024-11-11T03:17:35,505 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-11T03:17:35,505 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=74, ppid=73, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 276 msec 2024-11-11T03:17:35,507 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=73, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 408 msec 2024-11-11T03:17:35,516 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T03:17:35,531 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6ddd6177 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@46c961a4 2024-11-11T03:17:35,534 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:35,535 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:35,535 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@72d2cb74, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:35,536 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:35,539 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:35,539 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:35,539 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60040 connected 2024-11-11T03:17:35,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-11-11T03:17:35,726 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:35,727 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:35,728 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-11-11T03:17:35,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=75, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:35,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-11T03:17:35,746 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:35,746 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:35,749 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:35,749 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60041 connected 2024-11-11T03:17:35,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-11T03:17:35,851 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60041, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:35,855 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:36,007 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:36,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=76 2024-11-11T03:17:36,008 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:36,009 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:36,009 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:36,010 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:36,010 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60040, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:36,010 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60040, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-11T03:17:36,112 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60040, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:36,112 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=76}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6ddd6177 to 127.0.0.1:54139 2024-11-11T03:17:36,112 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=76}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:36,128 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:36,129 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:36,129 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=76 2024-11-11T03:17:36,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=76 2024-11-11T03:17:36,131 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:36,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=76, resume processing ppid=75 2024-11-11T03:17:36,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=76, ppid=75, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 276 msec 2024-11-11T03:17:36,133 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-11-11T03:17:36,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=75, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 405 msec 2024-11-11T03:17:36,160 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x74b5c75a to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7657b069 2024-11-11T03:17:36,163 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:36,164 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:36,164 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@15e6b496, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:36,165 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:36,168 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:36,168 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:36,168 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60043 connected 2024-11-11T03:17:36,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-11T03:17:36,356 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:36,357 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:36,358 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:36,359 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-11-11T03:17:36,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=77, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:36,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-11-11T03:17:36,381 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:36,381 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:36,390 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:36,391 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60044 connected 2024-11-11T03:17:36,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-11-11T03:17:36,492 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60044, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:36,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:36,647 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:36,648 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=78 2024-11-11T03:17:36,648 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:36,649 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:36,649 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:36,649 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:36,650 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60043, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:36,650 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60043, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:36,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-11-11T03:17:36,752 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60043, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:36,752 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=78}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x74b5c75a to 127.0.0.1:54139 2024-11-11T03:17:36,752 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=78}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:36,768 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:36,768 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:36,768 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=78 2024-11-11T03:17:36,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=78 2024-11-11T03:17:36,771 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:36,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=78, resume processing ppid=77 2024-11-11T03:17:36,773 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=78, ppid=77, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 275 msec 2024-11-11T03:17:36,773 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-11-11T03:17:36,774 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=77, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 414 msec 2024-11-11T03:17:36,801 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x343a263d to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6bd73e89 2024-11-11T03:17:36,804 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:36,805 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:36,805 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@660f1b96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:36,806 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:36,808 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:36,808 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:36,808 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60046 connected 2024-11-11T03:17:36,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-11-11T03:17:36,986 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:36,987 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:36,988 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:36,989 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-11-11T03:17:36,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=79, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:36,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-11-11T03:17:37,007 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:37,008 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:37,010 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:37,011 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60047 connected 2024-11-11T03:17:37,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-11-11T03:17:37,112 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60047, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:37,116 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:37,267 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:37,269 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=80 2024-11-11T03:17:37,269 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:37,270 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:37,270 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:37,271 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:37,271 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60046, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:37,271 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60046, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-11-11T03:17:37,373 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60046, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:37,373 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=80}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x343a263d to 127.0.0.1:54139 2024-11-11T03:17:37,373 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=80}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:37,389 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:37,389 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:37,390 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=80 2024-11-11T03:17:37,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=80 2024-11-11T03:17:37,393 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:37,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=80, resume processing ppid=79 2024-11-11T03:17:37,395 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=80, ppid=79, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 277 msec 2024-11-11T03:17:37,395 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-11-11T03:17:37,396 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=79, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 406 msec 2024-11-11T03:17:37,421 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x40812e2a to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3d5ee08a 2024-11-11T03:17:37,425 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:37,425 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:37,425 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@39ef4b25, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:37,426 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:37,429 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:37,429 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:37,429 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60049 connected 2024-11-11T03:17:37,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-11-11T03:17:37,616 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:37,617 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:37,617 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:37,619 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-11-11T03:17:37,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=81, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:37,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-11T03:17:37,638 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:37,638 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:37,644 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:37,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-11T03:17:37,744 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:37,748 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:37,900 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:37,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=82 2024-11-11T03:17:37,901 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:37,902 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:37,903 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:37,903 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:37,903 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60049, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:37,903 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60049, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:37,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-11T03:17:38,005 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60049, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:38,005 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=82}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x40812e2a to 127.0.0.1:54139 2024-11-11T03:17:38,005 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=82}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:38,021 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:38,021 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:38,022 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=82 2024-11-11T03:17:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=82 2024-11-11T03:17:38,024 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:38,026 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=82, resume processing ppid=81 2024-11-11T03:17:38,026 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=82, ppid=81, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 277 msec 2024-11-11T03:17:38,026 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-11-11T03:17:38,028 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=81, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 407 msec 2024-11-11T03:17:38,053 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2e57be12 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@423ac075 2024-11-11T03:17:38,056 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:38,056 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:38,057 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@56ab1a5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:38,057 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:38,059 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:38,060 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:38,060 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e6004c connected 2024-11-11T03:17:38,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-11T03:17:38,247 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:38,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:38,248 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:38,249 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-11-11T03:17:38,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=83, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:38,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-11-11T03:17:38,266 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:38,267 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:38,269 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:38,270 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6004d connected 2024-11-11T03:17:38,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-11-11T03:17:38,371 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6004d, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:38,375 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:38,527 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:38,527 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=84 2024-11-11T03:17:38,527 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:38,529 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:38,529 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:38,529 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:38,529 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e6004c, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:38,529 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e6004c, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:38,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-11-11T03:17:38,631 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e6004c, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:38,631 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=84}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2e57be12 to 127.0.0.1:54139 2024-11-11T03:17:38,631 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=84}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:38,655 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:38,655 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:38,656 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=84 2024-11-11T03:17:38,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=84 2024-11-11T03:17:38,659 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:38,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=84, resume processing ppid=83 2024-11-11T03:17:38,661 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=84, ppid=83, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 284 msec 2024-11-11T03:17:38,661 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-11-11T03:17:38,662 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=83, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 412 msec 2024-11-11T03:17:38,687 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x25c2b18a to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2434ad45 2024-11-11T03:17:38,691 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:38,691 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:38,691 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3a6043e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:38,692 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:38,694 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:38,695 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:38,695 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e6004f connected 2024-11-11T03:17:38,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-11-11T03:17:38,876 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:38,877 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:38,878 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-11-11T03:17:38,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=85, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:38,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-11-11T03:17:38,904 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:38,905 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:38,907 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:38,908 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60050 connected 2024-11-11T03:17:38,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-11-11T03:17:39,009 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60050, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:39,013 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:39,165 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:39,166 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=86 2024-11-11T03:17:39,166 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:39,167 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=86 2024-11-11T03:17:39,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=86 2024-11-11T03:17:39,169 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:39,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=86, resume processing ppid=85 2024-11-11T03:17:39,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=86, ppid=85, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 156 msec 2024-11-11T03:17:39,171 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-11-11T03:17:39,172 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=85, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 293 msec 2024-11-11T03:17:39,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-11-11T03:17:39,196 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:39,196 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:39,197 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:39,198 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-11-11T03:17:39,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=87, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:39,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-11T03:17:39,217 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:39,217 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:39,220 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:39,220 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60051 connected 2024-11-11T03:17:39,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-11T03:17:39,321 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60051, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:39,325 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:39,477 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:39,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=88 2024-11-11T03:17:39,478 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:39,479 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:39,479 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:39,479 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:39,480 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e6004f, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:39,480 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e6004f, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:39,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-11T03:17:39,581 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e6004f, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:39,582 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=88}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x25c2b18a to 127.0.0.1:54139 2024-11-11T03:17:39,582 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=88}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:39,599 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:39,600 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:39,604 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-11-11T03:17:39,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=88 2024-11-11T03:17:39,609 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:39,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=88, resume processing ppid=87 2024-11-11T03:17:39,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=88, ppid=87, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 284 msec 2024-11-11T03:17:39,612 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-11-11T03:17:39,614 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=87, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 414 msec 2024-11-11T03:17:39,642 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4ef919bc to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@36ef5892 2024-11-11T03:17:39,650 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:39,651 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:39,651 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@7bb1994f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:39,652 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:39,656 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:39,657 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:39,664 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60053 connected 2024-11-11T03:17:39,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-11T03:17:39,827 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:39,828 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:39,830 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-11-11T03:17:39,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=89, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:39,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-11-11T03:17:39,857 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:39,858 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:39,861 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-11-11T03:17:39,963 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:39,971 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:40,124 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:40,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=90 2024-11-11T03:17:40,125 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:40,127 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:40,127 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:40,127 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:40,127 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60053, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:40,128 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60053, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-11-11T03:17:40,230 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60053, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:40,230 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=90}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4ef919bc to 127.0.0.1:54139 2024-11-11T03:17:40,230 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=90}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:40,249 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:40,249 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:40,252 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-11T03:17:40,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=90 2024-11-11T03:17:40,255 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:40,258 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=90, resume processing ppid=89 2024-11-11T03:17:40,258 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-11-11T03:17:40,258 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=90, ppid=89, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 285 msec 2024-11-11T03:17:40,260 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=89, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 428 msec 2024-11-11T03:17:40,294 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x10888938 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@12da0b53 2024-11-11T03:17:40,298 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:40,299 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:40,299 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@299f7294, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:40,301 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:40,316 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:40,316 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:40,320 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60056 connected 2024-11-11T03:17:40,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-11-11T03:17:40,458 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:40,458 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:40,461 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:17:40,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=91, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:40,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91 2024-11-11T03:17:40,465 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-11T03:17:40,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-11T03:17:40,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:17:40,468 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:17:40,470 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:40,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91 2024-11-11T03:17:40,624 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:40,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=92 2024-11-11T03:17:40,626 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-11T03:17:40,626 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-11T03:17:40,626 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-11T03:17:40,626 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:40,626 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60056, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:40,627 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60056, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:40,729 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60056, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:40,729 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=92}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x10888938 to 127.0.0.1:54139 2024-11-11T03:17:40,729 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=92}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:40,729 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-11T03:17:40,731 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=92}] zookeeper.ZKUtil(111): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:17:40,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:17:40,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:17:40,751 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:17:40,754 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=92}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:17:40,754 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-11T03:17:40,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=92 2024-11-11T03:17:40,763 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:40,766 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=92, resume processing ppid=91 2024-11-11T03:17:40,766 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=92, ppid=91, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 293 msec 2024-11-11T03:17:40,770 DEBUG [PEWorker-4 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:17:40,770 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-11T03:17:40,773 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=91, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 310 msec 2024-11-11T03:17:40,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91 2024-11-11T03:17:40,776 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:17:40,777 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:17:40,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=93, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:40,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=93 2024-11-11T03:17:40,781 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:40,782 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=93, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:40,783 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=93, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-11-11T03:17:40,783 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:40,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=93 2024-11-11T03:17:40,887 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-11T03:17:40,888 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:17:40,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=94, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:40,890 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:40,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=94 2024-11-11T03:17:40,891 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=94, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:40,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=94, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-11-11T03:17:40,892 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:40,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=94 2024-11-11T03:17:40,999 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:17:41,022 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[0] Thread=219 (was 220), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=403 (was 383) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7571 (was 7723) 2024-11-11T03:17:41,042 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[0] Thread=219, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=403, ProcessCount=11, AvailableMemoryMB=7570 2024-11-11T03:17:41,044 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:17:41,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=95, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:17:41,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=95 2024-11-11T03:17:41,077 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:41,078 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:41,116 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:41,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=95 2024-11-11T03:17:41,210 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:41,223 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:41,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=95 2024-11-11T03:17:41,384 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:41,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=96 2024-11-11T03:17:41,385 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-11T03:17:41,409 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=96}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:41,432 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:41,439 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-11T03:17:41,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=96 2024-11-11T03:17:41,442 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:41,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=96, resume processing ppid=95 2024-11-11T03:17:41,446 INFO [PEWorker-3 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-11T03:17:41,446 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=96, ppid=95, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 219 msec 2024-11-11T03:17:41,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=95, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 402 msec 2024-11-11T03:17:41,483 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4dc8c2e1 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@303c98e9 2024-11-11T03:17:41,487 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:41,488 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:41,488 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@5fb1a136, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:41,489 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:41,504 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:41,508 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:41,516 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60059 connected 2024-11-11T03:17:41,618 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T03:17:41,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=95 2024-11-11T03:17:41,677 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:17:41,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-11-11T03:17:41,687 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3938): Client=jenkins//172.17.0.2 disable replication peer, id=1 2024-11-11T03:17:41,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=97, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure 2024-11-11T03:17:41,701 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:41,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=97 2024-11-11T03:17:41,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=97 2024-11-11T03:17:41,829 ERROR [LeaseRenewer:jenkins@localhost:35635 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:35635,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:41,855 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:41,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=98 2024-11-11T03:17:41,856 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=DISABLE_PEER 2024-11-11T03:17:41,857 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:41,857 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:41,857 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:41,858 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60059, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:41,858 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60059, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:41,962 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60059, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:41,962 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=98}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4dc8c2e1 to 127.0.0.1:54139 2024-11-11T03:17:41,962 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=98}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:41,980 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:41,980 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:41,980 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-11T03:17:41,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=98 2024-11-11T03:17:41,983 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for DISABLE on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:41,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=98, resume processing ppid=97 2024-11-11T03:17:41,985 INFO [PEWorker-4 {}] replication.DisablePeerProcedure(67): Successfully disabled peer 1 2024-11-11T03:17:41,985 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=98, ppid=97, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 282 msec 2024-11-11T03:17:41,986 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=97, state=SUCCESS; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure in 298 msec 2024-11-11T03:17:42,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=97 2024-11-11T03:17:42,016 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: DISABLE_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:17:42,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-11-11T03:17:42,021 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:17:42,021 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5f04fa7e to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5e0d97a1 2024-11-11T03:17:42,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=99, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:42,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=99 2024-11-11T03:17:42,034 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-11T03:17:42,038 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-11T03:17:42,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:17:42,039 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:17:42,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:42,050 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:42,050 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:42,051 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@53a5046d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:42,052 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:42,068 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:42,068 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:42,078 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e6005b connected 2024-11-11T03:17:42,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=99 2024-11-11T03:17:42,192 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:42,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=100 2024-11-11T03:17:42,194 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-11T03:17:42,194 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-11T03:17:42,194 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-11T03:17:42,194 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:42,194 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e6005b, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:42,195 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e6005b, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:42,298 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e6005b, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:42,298 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=100}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5f04fa7e to 127.0.0.1:54139 2024-11-11T03:17:42,298 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=100}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:42,299 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-11T03:17:42,301 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=100}] zookeeper.ZKUtil(111): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:17:42,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:17:42,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:17:42,310 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:17:42,314 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=100}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:17:42,314 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-11T03:17:42,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=100 2024-11-11T03:17:42,318 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:42,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=100, resume processing ppid=99 2024-11-11T03:17:42,322 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=100, ppid=99, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 278 msec 2024-11-11T03:17:42,332 DEBUG [PEWorker-5 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:17:42,332 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-11T03:17:42,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=99, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 311 msec 2024-11-11T03:17:42,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=99 2024-11-11T03:17:42,337 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:17:42,337 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:17:42,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=101, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:42,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=101 2024-11-11T03:17:42,340 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:42,341 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=101, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:42,342 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=101, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-11-11T03:17:42,342 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:42,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=101 2024-11-11T03:17:42,447 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-11T03:17:42,447 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:17:42,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=102, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:42,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=102 2024-11-11T03:17:42,450 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:42,451 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=102, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:42,453 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=102, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-11-11T03:17:42,453 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:42,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=102 2024-11-11T03:17:42,557 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:17:42,579 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[0] Thread=218 (was 219), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=403 (was 403), ProcessCount=11 (was 11), AvailableMemoryMB=7536 (was 7570) 2024-11-11T03:17:42,589 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[0] Thread=218, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=403, ProcessCount=11, AvailableMemoryMB=7536 2024-11-11T03:17:42,590 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=dummy_peer 2024-11-11T03:17:42,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=103, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:42,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=103 2024-11-11T03:17:42,597 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer dummy_peer, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:42,598 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=103, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:42,600 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=103, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=9 msec 2024-11-11T03:17:42,600 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:42,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=103 2024-11-11T03:17:42,707 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: dummy_peer failed with dummy_peer 2024-11-11T03:17:42,708 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:17:42,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=104, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=104 2024-11-11T03:17:42,711 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:42,712 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=104, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:42,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=104, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-11-11T03:17:42,717 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:42,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=104 2024-11-11T03:17:42,817 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-11T03:17:42,817 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:17:42,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=105, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:42,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=105 2024-11-11T03:17:42,828 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:42,830 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=105, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:42,831 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=105, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=12 msec 2024-11-11T03:17:42,831 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:42,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=105 2024-11-11T03:17:42,927 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:17:42,945 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[0] Thread=218 (was 218), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=403 (was 403), ProcessCount=11 (was 11), AvailableMemoryMB=7534 (was 7536) 2024-11-11T03:17:42,959 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerConfig[0] Thread=218, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=403, ProcessCount=11, AvailableMemoryMB=7534 2024-11-11T03:17:42,961 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:17:42,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=106, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:17:42,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-11-11T03:17:42,995 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:42,995 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:42,999 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:43,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-11-11T03:17:43,102 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:43,109 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:43,275 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:43,276 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=107 2024-11-11T03:17:43,276 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-11T03:17:43,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-11-11T03:17:43,295 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=107}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:43,315 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:43,325 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=107 2024-11-11T03:17:43,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=107 2024-11-11T03:17:43,327 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:43,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=107, resume processing ppid=106 2024-11-11T03:17:43,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=107, ppid=106, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 218 msec 2024-11-11T03:17:43,329 INFO [PEWorker-2 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-11T03:17:43,331 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=106, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 368 msec 2024-11-11T03:17:43,362 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2270dde4 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3b9e2ada 2024-11-11T03:17:43,366 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:43,366 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:43,366 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@7f1d4dd2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:43,369 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:43,372 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:43,372 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:43,376 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e6005e connected 2024-11-11T03:17:43,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-11-11T03:17:43,597 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:17:43,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-11-11T03:17:43,598 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:17:43,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=108, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:43,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-11-11T03:17:43,604 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-11T03:17:43,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-11T03:17:43,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:17:43,607 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:17:43,608 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:43,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-11-11T03:17:43,760 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:43,761 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=109 2024-11-11T03:17:43,761 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-11T03:17:43,762 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-11T03:17:43,762 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-11T03:17:43,762 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:43,762 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e6005e, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:43,762 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e6005e, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:43,865 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e6005e, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:43,865 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=109}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2270dde4 to 127.0.0.1:54139 2024-11-11T03:17:43,865 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=109}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:43,865 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-11T03:17:43,867 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=109}] zookeeper.ZKUtil(111): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:17:43,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:17:43,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:17:43,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:17:43,870 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=109}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:17:43,870 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=109 2024-11-11T03:17:43,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=109 2024-11-11T03:17:43,872 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:43,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=109, resume processing ppid=108 2024-11-11T03:17:43,874 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=109, ppid=108, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 264 msec 2024-11-11T03:17:43,877 DEBUG [PEWorker-1 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:17:43,877 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-11T03:17:43,878 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=108, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 278 msec 2024-11-11T03:17:43,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-11-11T03:17:43,916 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:17:43,916 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:17:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=110, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:43,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-11-11T03:17:43,918 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:43,919 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=110, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:43,920 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=110, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-11-11T03:17:43,920 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:44,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-11-11T03:17:44,026 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-11T03:17:44,027 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:17:44,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=111, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:44,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=111 2024-11-11T03:17:44,029 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:44,030 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=111, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:44,031 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=111, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-11-11T03:17:44,031 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:44,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=111 2024-11-11T03:17:44,136 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:17:44,151 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerConfig[0] Thread=218 (was 218), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=403 (was 403), ProcessCount=11 (was 11), AvailableMemoryMB=7515 (was 7534) 2024-11-11T03:17:44,161 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[0] Thread=218, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=403, ProcessCount=11, AvailableMemoryMB=7515 2024-11-11T03:17:44,168 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:17:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-11T03:17:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] ipc.CallRunner(138): callId: 342 service: MasterService methodName: AddReplicationPeer size: 80 connection: 172.17.0.2:39256 deadline: 1731295124168, exception=java.io.IOException: Replication peer modification disabled 2024-11-11T03:17:44,169 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(168): Call to master failed, tries = 1, maxAttempts = 2, timeout = 120000 ms, time elapsed = 1 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:245) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:431) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 33 more 2024-11-11T03:17:44,276 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:17:44,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-11T03:17:44,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] ipc.CallRunner(138): callId: 343 service: MasterService methodName: AddReplicationPeer size: 80 connection: 172.17.0.2:39256 deadline: 1731295124275, exception=java.io.IOException: Replication peer modification disabled 2024-11-11T03:17:44,277 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(168): Call to master failed, tries = 2, maxAttempts = 2, timeout = 120000 ms, time elapsed = 109 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:245) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:431) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 33 more 2024-11-11T03:17:44,278 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Failed after attempts=2, exceptions: 2024-11-11T03:17:44.171Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) 2024-11-11T03:17:44.277Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) 2024-11-11T03:17:44,280 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:17:44,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=112, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:44,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-11-11T03:17:44,282 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:44,283 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=112, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:44,284 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=112, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-11-11T03:17:44,284 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:44,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-11-11T03:17:44,387 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-11T03:17:44,387 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:17:44,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=113, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:44,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=113 2024-11-11T03:17:44,390 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:44,391 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=113, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:44,392 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=113, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-11-11T03:17:44,392 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:44,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=113 2024-11-11T03:17:44,497 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:17:44,512 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[0] Thread=218 (was 218), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=403 (was 403), ProcessCount=11 (was 11), AvailableMemoryMB=7513 (was 7515) 2024-11-11T03:17:44,523 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[0] Thread=218, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=403, ProcessCount=11, AvailableMemoryMB=7513 2024-11-11T03:17:44,524 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:17:44,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=114, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:17:44,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-11-11T03:17:44,552 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:44,552 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:44,579 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:44,584 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6005f connected 2024-11-11T03:17:44,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-11-11T03:17:44,686 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6005f, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:44,693 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:44,845 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:44,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-11-11T03:17:44,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=115 2024-11-11T03:17:44,846 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-11T03:17:44,866 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=115}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:44,883 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:44,889 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=115 2024-11-11T03:17:44,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=115 2024-11-11T03:17:44,892 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:44,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=115, resume processing ppid=114 2024-11-11T03:17:44,894 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=115, ppid=114, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 200 msec 2024-11-11T03:17:44,894 INFO [PEWorker-5 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-11T03:17:44,895 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=114, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 370 msec 2024-11-11T03:17:44,932 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5c0742da to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@48f0d736 2024-11-11T03:17:44,951 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:44,951 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:44,951 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@29457373, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:44,953 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:44,956 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:44,956 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:44,957 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60061 connected 2024-11-11T03:17:45,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-11-11T03:17:45,157 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:17:45,157 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:45,158 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:17:45,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=116, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:45,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116 2024-11-11T03:17:45,165 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-11T03:17:45,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-11T03:17:45,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:17:45,167 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:17:45,168 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:45,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116 2024-11-11T03:17:45,320 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:45,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=117 2024-11-11T03:17:45,322 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-11T03:17:45,322 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-11T03:17:45,322 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-11T03:17:45,322 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:45,322 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60061, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:45,322 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60061, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:45,424 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60061, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:45,424 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=117}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5c0742da to 127.0.0.1:54139 2024-11-11T03:17:45,425 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=117}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:45,425 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-11T03:17:45,428 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=117}] zookeeper.ZKUtil(111): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:17:45,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:17:45,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:17:45,431 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:17:45,431 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=117}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:17:45,431 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=117 2024-11-11T03:17:45,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=117 2024-11-11T03:17:45,434 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:45,437 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=117, resume processing ppid=116 2024-11-11T03:17:45,437 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=117, ppid=116, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 266 msec 2024-11-11T03:17:45,441 DEBUG [PEWorker-4 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:17:45,441 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-11T03:17:45,443 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=116, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 283 msec 2024-11-11T03:17:45,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116 2024-11-11T03:17:45,476 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:17:45,476 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:17:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=118, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:45,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=118 2024-11-11T03:17:45,479 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:45,480 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=118, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:45,481 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=118, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-11-11T03:17:45,481 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:45,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=118 2024-11-11T03:17:45,586 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:17:45,603 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[0] Thread=218 (was 218), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=443 (was 403) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7498 (was 7513) 2024-11-11T03:17:45,615 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerBandwidth[0] Thread=218, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=443, ProcessCount=11, AvailableMemoryMB=7497 2024-11-11T03:17:45,616 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:17:45,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=119, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:17:45,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119 2024-11-11T03:17:45,643 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:45,645 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:45,648 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:45,649 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60062 connected 2024-11-11T03:17:45,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119 2024-11-11T03:17:45,751 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60062, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:45,758 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:45,911 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:45,911 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=120 2024-11-11T03:17:45,912 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-11T03:17:45,936 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=120}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:45,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119 2024-11-11T03:17:45,955 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:45,965 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-11T03:17:45,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=120 2024-11-11T03:17:45,967 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:45,969 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=120, resume processing ppid=119 2024-11-11T03:17:45,969 INFO [PEWorker-4 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-11T03:17:45,969 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=120, ppid=119, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 209 msec 2024-11-11T03:17:45,970 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=119, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 352 msec 2024-11-11T03:17:46,002 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x192f1f85 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@499b22 2024-11-11T03:17:46,006 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:46,006 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:46,007 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@79f4ff87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:46,008 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:46,014 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:46,015 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:46,015 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60064 connected 2024-11-11T03:17:46,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119 2024-11-11T03:17:46,247 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:17:46,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:46,249 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-11-11T03:17:46,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=121, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=121 2024-11-11T03:17:46,257 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T03:17:46,273 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:46,274 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:46,288 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:46,289 ERROR [PEWorker-5-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$361/0x00007f778c9134e8@73dc35dd rejected from java.util.concurrent.ThreadPoolExecutor@36deb5c2[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 0] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.PendingWatcher.process(PendingWatcher.java:38) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-11-11T03:17:46,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=121 2024-11-11T03:17:46,394 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:46,399 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:46,551 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:46,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=122 2024-11-11T03:17:46,552 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:46,553 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-11T03:17:46,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=122 2024-11-11T03:17:46,555 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:46,557 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=122, resume processing ppid=121 2024-11-11T03:17:46,557 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-11-11T03:17:46,558 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=122, ppid=121, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 157 msec 2024-11-11T03:17:46,559 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=121, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 309 msec 2024-11-11T03:17:46,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=121 2024-11-11T03:17:46,576 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:46,577 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:46,577 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:17:46,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=123, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:46,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=123 2024-11-11T03:17:46,581 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-11T03:17:46,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-11T03:17:46,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:17:46,583 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:17:46,584 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:46,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=123 2024-11-11T03:17:46,736 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:46,736 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=124 2024-11-11T03:17:46,737 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-11T03:17:46,737 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-11T03:17:46,737 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-11T03:17:46,737 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:46,737 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60064, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:46,738 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60064, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:46,828 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:35635 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:35635,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:46,839 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60064, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:46,840 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=124}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x192f1f85 to 127.0.0.1:54139 2024-11-11T03:17:46,840 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=124}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:46,840 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-11T03:17:46,841 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=124}] zookeeper.ZKUtil(111): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:17:46,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:17:46,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:17:46,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:17:46,845 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=124}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:17:46,845 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-11T03:17:46,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=124 2024-11-11T03:17:46,847 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:46,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=124, resume processing ppid=123 2024-11-11T03:17:46,849 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=124, ppid=123, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 263 msec 2024-11-11T03:17:46,859 DEBUG [PEWorker-3 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:17:46,859 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-11T03:17:46,860 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=123, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 282 msec 2024-11-11T03:17:46,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=123 2024-11-11T03:17:46,896 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:17:46,896 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:17:46,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=125, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:46,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=125 2024-11-11T03:17:46,898 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:46,899 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=125, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:46,900 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=125, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-11-11T03:17:46,900 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:46,924 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-11T03:17:46,927 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34128, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-11T03:17:47,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=125 2024-11-11T03:17:47,007 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-11T03:17:47,007 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:17:47,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=126, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:47,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=126 2024-11-11T03:17:47,009 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:47,010 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=126, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:47,011 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=126, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-11-11T03:17:47,011 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:47,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=126 2024-11-11T03:17:47,117 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:17:47,130 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerBandwidth[0] Thread=217 (was 218), OpenFileDescriptor=446 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=443 (was 443), ProcessCount=11 (was 11), AvailableMemoryMB=7460 (was 7497) 2024-11-11T03:17:47,140 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[0] Thread=217, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=443, ProcessCount=11, AvailableMemoryMB=7459 2024-11-11T03:17:47,141 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:17:47,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=127, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:17:47,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=127 2024-11-11T03:17:47,149 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:47,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=127 2024-11-11T03:17:47,301 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:47,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=128 2024-11-11T03:17:47,302 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=128}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-11T03:17:47,345 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=128}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:47,354 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-11T03:17:47,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=128 2024-11-11T03:17:47,355 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(587): peerId=1, queueId=1 (queues=1) is replicating from cluster=454cc11f-9833-4fa8-a013-fffc9bade1b4 to cluster=454cc11f-9833-4fa8-a013-fffc9bade1b4 2024-11-11T03:17:47,357 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(343): peerId=1, starting shipping worker for walGroupId=e4cdd3a0b883%2C41101%2C1731295033876 2024-11-11T03:17:47,358 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:47,360 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSourceWALReader(112): peerClusterZnode=1, ReplicationSourceWALReaderThread : 1 inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-11T03:17:47,360 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=128, resume processing ppid=127 2024-11-11T03:17:47,360 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=128, ppid=127, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 209 msec 2024-11-11T03:17:47,360 INFO [PEWorker-2 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-11T03:17:47,362 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1.replicationSource.wal-reader.e4cdd3a0b883%2C41101%2C1731295033876,1 {}] regionserver.WALEntryStream(260): Creating new reader hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/WALs/e4cdd3a0b883,41101,1731295033876/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784, startPosition=0, beingWritten=true 2024-11-11T03:17:47,362 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1.replicationSource.shippere4cdd3a0b883%2C41101%2C1731295033876,1 {}] regionserver.ReplicationSourceShipper(98): Running ReplicationSourceShipper Thread for wal group: e4cdd3a0b883%2C41101%2C1731295033876 2024-11-11T03:17:47,364 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=127, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 219 msec 2024-11-11T03:17:47,445 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1.replicationSource.wal-reader.e4cdd3a0b883%2C41101%2C1731295033876,1 {}] regionserver.ReplicationSourceWALReader(181): Read 0 WAL entries eligible for replication 2024-11-11T03:17:47,445 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1.replicationSource.wal-reader.e4cdd3a0b883%2C41101%2C1731295033876,1 {}] regionserver.WALEntryStream(223): Reset reader hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/WALs/e4cdd3a0b883,41101,1731295033876/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 to pos 589, reset compression=false 2024-11-11T03:17:47,445 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1.replicationSource.shippere4cdd3a0b883%2C41101%2C1731295033876,1 {}] regionserver.ReplicationSourceShipper(110): Shipper from source 1 got entry batch from reader: WALEntryBatch [walEntries=[], lastWalPath=hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/WALs/e4cdd3a0b883,41101,1731295033876/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784, lastWalPosition=589, nbRowKeys=0, nbHFiles=0, heapSize=0, lastSeqIds={}, endOfFile=false,usedBufferSize=0] 2024-11-11T03:17:47,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=127 2024-11-11T03:17:47,456 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:17:47,457 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=2, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:17:47,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=129, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:17:47,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=129 2024-11-11T03:17:47,459 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] ... 13 more 2024-11-11T03:17:47,460 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=129, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:47,461 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=129, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=3 msec 2024-11-11T03:17:47,461 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:47,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=129 2024-11-11T03:17:47,566 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 2 failed with Invalid cluster key: 2024-11-11T03:17:47,567 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:17:47,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=130, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:47,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-11T03:17:47,570 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-11T03:17:47,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-11T03:17:47,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:17:47,573 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:17:47,573 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:47,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-11T03:17:47,725 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:47,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=131 2024-11-11T03:17:47,726 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-11T03:17:47,726 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-11T03:17:47,726 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-11T03:17:47,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-11T03:17:48,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-11T03:17:48,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-11T03:17:48,726 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1.replicationSource.wal-reader.e4cdd3a0b883%2C41101%2C1731295033876,1 {}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:131) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:166) ~[classes/:?] 2024-11-11T03:17:48,726 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1.replicationSource.shippere4cdd3a0b883%2C41101%2C1731295033876,1 {}] regionserver.ReplicationSourceShipper(123): Interrupted while waiting for next replication entry batch java.lang.InterruptedException: null at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1681) ~[?:?] at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) ~[?:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.poll(ReplicationSourceWALReader.java:313) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceShipper.run(ReplicationSourceShipper.java:109) ~[classes/:?] 2024-11-11T03:17:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-11T03:17:49,726 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.ReplicationSource(711): peerId=1, ReplicationSourceWorker RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1.replicationSource.shippere4cdd3a0b883%2C41101%2C1731295033876,1 terminated 2024-11-11T03:17:49,727 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-11T03:17:49,728 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=131}] zookeeper.ZKUtil(111): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:17:49,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:17:49,730 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:17:49,731 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:17:49,731 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=131}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:17:49,731 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=131 2024-11-11T03:17:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=131 2024-11-11T03:17:49,733 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:49,734 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=131, resume processing ppid=130 2024-11-11T03:17:49,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=131, ppid=130, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 2.1600 sec 2024-11-11T03:17:49,737 DEBUG [PEWorker-2 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:17:49,737 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-11T03:17:49,739 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=130, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 2.1700 sec 2024-11-11T03:17:51,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-11T03:17:51,737 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:17:51,738 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:17:51,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=132, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:51,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-11-11T03:17:51,740 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:51,742 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=132, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:51,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=132, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-11-11T03:17:51,743 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:51,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-11-11T03:17:51,847 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:17:51,864 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[0] Thread=218 (was 217) - Thread LEAK? -, OpenFileDescriptor=451 (was 446) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=448 (was 443) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7424 (was 7459) 2024-11-11T03:17:51,880 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[1] Thread=218, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=448, ProcessCount=11, AvailableMemoryMB=7423 2024-11-11T03:17:51,884 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:17:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=133, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:17:51,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=133 2024-11-11T03:17:51,914 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:51,915 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:51,930 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:51,933 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60066 connected 2024-11-11T03:17:51,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=133 2024-11-11T03:17:52,040 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60066, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:52,051 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:52,204 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:52,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=134 2024-11-11T03:17:52,205 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-11T03:17:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=133 2024-11-11T03:17:52,227 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=134}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:52,246 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:52,258 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-11T03:17:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=134 2024-11-11T03:17:52,261 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:52,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=134, resume processing ppid=133 2024-11-11T03:17:52,265 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=134, ppid=133, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 210 msec 2024-11-11T03:17:52,265 INFO [PEWorker-3 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-11T03:17:52,267 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=133, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 381 msec 2024-11-11T03:17:52,305 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x21f2c525 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1853b8e1 2024-11-11T03:17:52,320 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:52,321 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:52,321 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3b2f2ff2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:52,323 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:52,326 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:52,328 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:52,334 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60068 connected 2024-11-11T03:17:52,343 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T03:17:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=133 2024-11-11T03:17:52,516 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:17:52,517 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-11T03:17:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=135, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=135 2024-11-11T03:17:52,542 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:52,543 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:52,569 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:52,570 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60069 connected 2024-11-11T03:17:52,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=135 2024-11-11T03:17:52,672 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60069, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:52,676 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:52,828 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:52,828 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=136 2024-11-11T03:17:52,829 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:52,830 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:52,830 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:52,830 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:52,830 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60068, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:52,831 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60068, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=135 2024-11-11T03:17:52,933 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60068, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:52,933 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=136}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x21f2c525 to 127.0.0.1:54139 2024-11-11T03:17:52,933 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=136}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:52,951 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:52,952 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:52,952 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-11T03:17:52,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=136 2024-11-11T03:17:52,955 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:52,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=136, resume processing ppid=135 2024-11-11T03:17:52,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=136, ppid=135, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 279 msec 2024-11-11T03:17:52,957 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-11T03:17:52,959 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=135, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 440 msec 2024-11-11T03:17:52,996 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3be02be9 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4bcc8afa 2024-11-11T03:17:53,018 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:53,018 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:53,019 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@586937b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:53,020 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:53,034 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:53,035 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:53,041 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e6006b connected 2024-11-11T03:17:53,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=135 2024-11-11T03:17:53,146 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:53,147 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:53,148 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:53,149 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:53,150 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-11-11T03:17:53,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=137, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:53,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=137 2024-11-11T03:17:53,174 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:53,175 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:53,184 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:53,188 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6006c connected 2024-11-11T03:17:53,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=137 2024-11-11T03:17:53,293 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6006c, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:53,297 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:53,449 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:53,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=138 2024-11-11T03:17:53,450 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:53,451 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:53,451 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:53,451 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:53,452 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e6006b, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:53,452 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e6006b, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:53,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=137 2024-11-11T03:17:53,554 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e6006b, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:53,554 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=138}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3be02be9 to 127.0.0.1:54139 2024-11-11T03:17:53,554 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=138}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:53,577 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:53,577 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:53,577 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-11T03:17:53,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=138 2024-11-11T03:17:53,580 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:53,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=138, resume processing ppid=137 2024-11-11T03:17:53,583 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-11-11T03:17:53,583 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=138, ppid=137, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 283 msec 2024-11-11T03:17:53,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=137, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 433 msec 2024-11-11T03:17:53,620 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x66a69032 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3d502366 2024-11-11T03:17:53,624 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:53,625 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:53,625 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3466d237, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:53,626 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:53,629 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:53,629 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:53,629 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e6006e connected 2024-11-11T03:17:53,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=137 2024-11-11T03:17:53,777 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:53,777 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:53,778 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:53,779 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:53,780 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:53,780 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-11-11T03:17:53,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=139, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:53,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139 2024-11-11T03:17:53,802 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:53,803 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:53,806 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:53,806 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6006f connected 2024-11-11T03:17:53,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139 2024-11-11T03:17:53,908 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6006f, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:53,912 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:54,064 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:54,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=140 2024-11-11T03:17:54,065 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:54,066 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:54,066 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:54,066 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:54,066 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e6006e, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:54,067 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e6006e, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:54,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139 2024-11-11T03:17:54,169 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e6006e, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:54,169 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=140}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x66a69032 to 127.0.0.1:54139 2024-11-11T03:17:54,169 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=140}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:54,186 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:54,186 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:54,186 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-11T03:17:54,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=140 2024-11-11T03:17:54,189 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:54,191 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=140, resume processing ppid=139 2024-11-11T03:17:54,191 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-11-11T03:17:54,191 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=140, ppid=139, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 278 msec 2024-11-11T03:17:54,192 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=139, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 411 msec 2024-11-11T03:17:54,221 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x09c51b11 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1c325caf 2024-11-11T03:17:54,225 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:54,225 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:54,225 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@7e2d607f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:54,226 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:54,229 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:54,229 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:54,229 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60071 connected 2024-11-11T03:17:54,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139 2024-11-11T03:17:54,406 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:54,407 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:54,407 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:54,408 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:54,409 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-11T03:17:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=141, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:54,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=141 2024-11-11T03:17:54,431 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:54,431 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:54,434 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:54,435 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60072 connected 2024-11-11T03:17:54,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=141 2024-11-11T03:17:54,537 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60072, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:54,540 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:54,692 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:54,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=142 2024-11-11T03:17:54,693 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:54,694 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:54,694 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:54,695 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:54,695 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60071, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:54,695 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60071, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=141 2024-11-11T03:17:54,797 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60071, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:54,797 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=142}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x09c51b11 to 127.0.0.1:54139 2024-11-11T03:17:54,797 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=142}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:54,867 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:54,867 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:54,868 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-11T03:17:54,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=142 2024-11-11T03:17:54,870 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:54,872 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=142, resume processing ppid=141 2024-11-11T03:17:54,872 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-11T03:17:54,872 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=142, ppid=141, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 330 msec 2024-11-11T03:17:54,873 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=141, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 463 msec 2024-11-11T03:17:54,906 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1d247933 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2e15001f 2024-11-11T03:17:54,910 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:54,911 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:54,911 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@40e392c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:54,912 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:54,914 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:54,915 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:54,918 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60074 connected 2024-11-11T03:17:55,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=141 2024-11-11T03:17:55,036 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:55,036 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:55,037 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:55,038 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-11-11T03:17:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=143, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:55,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143 2024-11-11T03:17:55,062 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:55,063 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:55,066 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:55,066 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60075 connected 2024-11-11T03:17:55,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143 2024-11-11T03:17:55,168 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60075, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:55,171 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:55,323 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:55,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=144 2024-11-11T03:17:55,324 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:55,325 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:55,325 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:55,325 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:55,326 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60074, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:55,326 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60074, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:55,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143 2024-11-11T03:17:55,428 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60074, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:55,428 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=144}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1d247933 to 127.0.0.1:54139 2024-11-11T03:17:55,428 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=144}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:55,448 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:55,448 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:55,448 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-11-11T03:17:55,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=144 2024-11-11T03:17:55,451 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:55,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=144, resume processing ppid=143 2024-11-11T03:17:55,453 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-11-11T03:17:55,453 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=144, ppid=143, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 280 msec 2024-11-11T03:17:55,454 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=143, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 415 msec 2024-11-11T03:17:55,487 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6cccf57f to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2addaea4 2024-11-11T03:17:55,491 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:55,491 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:55,492 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@58f3d871, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:55,493 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:55,495 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:55,495 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:55,495 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60077 connected 2024-11-11T03:17:55,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143 2024-11-11T03:17:55,666 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:55,666 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:55,668 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-11T03:17:55,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=145, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:55,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=145 2024-11-11T03:17:55,687 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:55,688 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:55,691 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:55,691 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60078 connected 2024-11-11T03:17:55,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=145 2024-11-11T03:17:55,793 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60078, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:55,797 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:55,949 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:55,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=146 2024-11-11T03:17:55,950 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:55,951 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:55,951 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:55,951 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:55,951 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60077, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:55,952 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60077, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:55,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=145 2024-11-11T03:17:56,054 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60077, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:56,054 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=146}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6cccf57f to 127.0.0.1:54139 2024-11-11T03:17:56,054 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=146}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:56,074 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:56,074 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:56,074 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=146 2024-11-11T03:17:56,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=146 2024-11-11T03:17:56,076 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:56,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=146, resume processing ppid=145 2024-11-11T03:17:56,078 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=146, ppid=145, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 279 msec 2024-11-11T03:17:56,078 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-11T03:17:56,080 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=145, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 411 msec 2024-11-11T03:17:56,110 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x104f3b17 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@169c4713 2024-11-11T03:17:56,113 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:56,114 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:56,114 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@1aa3fee2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:56,115 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:56,117 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:56,118 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:56,118 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e6007a connected 2024-11-11T03:17:56,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=145 2024-11-11T03:17:56,306 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:56,307 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:56,307 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:17:56,307 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:17:56,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=147, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=148, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-11T03:17:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-11-11T03:17:56,311 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-11T03:17:56,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-11T03:17:56,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:17:56,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:17:56,314 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-11-11T03:17:56,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-11T03:17:56,466 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:56,467 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=149 2024-11-11T03:17:56,467 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-11T03:17:56,467 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-11T03:17:56,468 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-11T03:17:56,468 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:56,468 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e6007a, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:56,468 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e6007a, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:56,570 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e6007a, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:56,570 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=149}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x104f3b17 to 127.0.0.1:54139 2024-11-11T03:17:56,570 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=149}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:56,571 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-11T03:17:56,572 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=149}] zookeeper.ZKUtil(111): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:17:56,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:17:56,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:17:56,575 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:17:56,575 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=149}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:17:56,575 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-11T03:17:56,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=149 2024-11-11T03:17:56,578 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:56,579 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=149, resume processing ppid=147 2024-11-11T03:17:56,579 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=149, ppid=147, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 264 msec 2024-11-11T03:17:56,582 DEBUG [PEWorker-2 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:17:56,582 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-11T03:17:56,583 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=147, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 275 msec 2024-11-11T03:17:56,583 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:56,584 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=148, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:56,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=148, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=277 msec 2024-11-11T03:17:56,585 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-11-11T03:17:56,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-11T03:17:56,634 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:17:56,635 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-11T03:17:56,638 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:17:56,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=150, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:56,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=150 2024-11-11T03:17:56,639 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:56,640 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=150, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:56,641 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=150, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-11-11T03:17:56,641 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:56,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=150 2024-11-11T03:17:56,747 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:17:56,764 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[1] Thread=220 (was 218) Potentially hanging thread: ForkJoinPool.commonPool-worker-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) - Thread LEAK? -, OpenFileDescriptor=450 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=532 (was 448) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=7298 (was 7423) 2024-11-11T03:17:56,777 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[1] Thread=220, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=532, ProcessCount=11, AvailableMemoryMB=7297 2024-11-11T03:17:56,778 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:17:56,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=151, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:17:56,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-11-11T03:17:56,800 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:56,800 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:56,803 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:56,804 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6007b connected 2024-11-11T03:17:56,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-11-11T03:17:56,906 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6007b, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:56,915 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:57,066 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:57,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=152 2024-11-11T03:17:57,067 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-11T03:17:57,087 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=152}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:57,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-11-11T03:17:57,108 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:57,115 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=152 2024-11-11T03:17:57,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=152 2024-11-11T03:17:57,117 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:57,119 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=152, resume processing ppid=151 2024-11-11T03:17:57,119 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=152, ppid=151, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 202 msec 2024-11-11T03:17:57,119 INFO [PEWorker-2 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-11T03:17:57,120 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=151, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 341 msec 2024-11-11T03:17:57,153 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x26ad75b8 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@248a151c 2024-11-11T03:17:57,159 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:57,160 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:57,160 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@5681bbc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:57,161 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:57,167 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:57,168 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:57,168 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e6007d connected 2024-11-11T03:17:57,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-11-11T03:17:57,416 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:17:57,417 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-11T03:17:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=153, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:57,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-11-11T03:17:57,441 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:57,442 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:57,448 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:57,449 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6007e connected 2024-11-11T03:17:57,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-11-11T03:17:57,551 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6007e, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:57,555 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:57,707 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:57,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=154 2024-11-11T03:17:57,708 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:57,709 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:57,709 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:57,709 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:57,710 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e6007d, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:57,710 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e6007d, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:57,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-11-11T03:17:57,812 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e6007d, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:57,812 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=154}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x26ad75b8 to 127.0.0.1:54139 2024-11-11T03:17:57,812 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=154}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:57,834 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:57,834 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:57,834 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=154 2024-11-11T03:17:57,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=154 2024-11-11T03:17:57,849 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:57,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=154, resume processing ppid=153 2024-11-11T03:17:57,851 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-11T03:17:57,851 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=154, ppid=153, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 294 msec 2024-11-11T03:17:57,855 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=153, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 434 msec 2024-11-11T03:17:57,877 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x54cc1f1a to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@ac7e67f 2024-11-11T03:17:57,882 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:57,883 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:57,883 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@7f618300, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:57,884 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:57,888 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:57,888 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:57,889 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60080 connected 2024-11-11T03:17:58,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-11-11T03:17:58,046 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:58,047 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:58,048 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-11-11T03:17:58,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=155, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:58,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-11-11T03:17:58,065 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:58,066 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:58,069 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:58,069 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60081 connected 2024-11-11T03:17:58,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-11-11T03:17:58,171 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60081, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:58,174 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:58,326 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:58,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=156 2024-11-11T03:17:58,327 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=156}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:58,328 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=156}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:58,328 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=156}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:58,328 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:58,328 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60080, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:58,329 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60080, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:58,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-11-11T03:17:58,431 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60080, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:58,431 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=156}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x54cc1f1a to 127.0.0.1:54139 2024-11-11T03:17:58,431 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=156}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:58,450 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=156}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:58,451 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=156}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:58,451 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=156}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=156 2024-11-11T03:17:58,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=156 2024-11-11T03:17:58,453 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:58,455 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=156, resume processing ppid=155 2024-11-11T03:17:58,455 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=156, ppid=155, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 279 msec 2024-11-11T03:17:58,455 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-11-11T03:17:58,456 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=155, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 407 msec 2024-11-11T03:17:58,491 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x31c9796c to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@87e77a8 2024-11-11T03:17:58,494 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:58,494 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:58,495 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@22555b82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:58,496 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:58,498 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:58,498 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:58,498 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60083 connected 2024-11-11T03:17:58,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-11-11T03:17:58,676 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:58,677 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:58,678 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns1:testNamespacesAndTableCfsConfigConflict1=null},bandwidth=0,serial=false 2024-11-11T03:17:58,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=157, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:58,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-11-11T03:17:58,696 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:58,697 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:58,699 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:58,700 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60084 connected 2024-11-11T03:17:58,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-11-11T03:17:58,802 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60084, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:58,803 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:58,804 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=157, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:58,805 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=157, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=126 msec 2024-11-11T03:17:58,805 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:58,957 INFO [master/e4cdd3a0b883:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-11T03:17:58,957 INFO [master/e4cdd3a0b883:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-11T03:17:58,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-11-11T03:17:58,996 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config 2024-11-11T03:17:58,997 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:58,998 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-11-11T03:17:58,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=158, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:59,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=158 2024-11-11T03:17:59,021 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:59,023 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:59,027 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:59,044 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60085 connected 2024-11-11T03:17:59,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=158 2024-11-11T03:17:59,147 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60085, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:59,152 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:17:59,304 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:17:59,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=159 2024-11-11T03:17:59,305 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=159}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:17:59,306 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=159}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:17:59,306 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=159}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:17:59,307 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:17:59,307 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60083, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:59,307 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60083, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:17:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=158 2024-11-11T03:17:59,409 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60083, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:59,409 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=159}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x31c9796c to 127.0.0.1:54139 2024-11-11T03:17:59,410 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=159}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:17:59,433 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=159}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:17:59,433 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=159}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:17:59,434 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-11T03:17:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=159 2024-11-11T03:17:59,437 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:17:59,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=159, resume processing ppid=158 2024-11-11T03:17:59,439 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-11-11T03:17:59,439 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=159, ppid=158, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 285 msec 2024-11-11T03:17:59,440 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=158, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 441 msec 2024-11-11T03:17:59,483 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6738149a to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@15fed3e6 2024-11-11T03:17:59,520 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:17:59,521 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:17:59,521 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@29876c05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:17:59,522 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:59,544 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:59,544 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:17:59,552 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60087 connected 2024-11-11T03:17:59,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=158 2024-11-11T03:17:59,627 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:17:59,627 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:17:59,628 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-11-11T03:17:59,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=160, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:17:59,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=160 2024-11-11T03:17:59,650 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:17:59,651 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:17:59,653 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:17:59,654 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60088 connected 2024-11-11T03:17:59,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=160 2024-11-11T03:17:59,755 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60088, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:17:59,755 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:17:59,757 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=160, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:17:59,758 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=160, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=129 msec 2024-11-11T03:17:59,758 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:17:59,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=160 2024-11-11T03:17:59,946 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config 2024-11-11T03:17:59,946 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:17:59,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=161, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:17:59,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=161 2024-11-11T03:17:59,950 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-11T03:17:59,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-11T03:17:59,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:17:59,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:17:59,953 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:00,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=161 2024-11-11T03:18:00,104 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:00,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=162 2024-11-11T03:18:00,105 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=162}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-11T03:18:00,105 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=162}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-11T03:18:00,105 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=162}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-11T03:18:00,106 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:18:00,106 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60087, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:00,106 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60087, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:00,208 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60087, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:00,208 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=162}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6738149a to 127.0.0.1:54139 2024-11-11T03:18:00,208 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=162}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:18:00,209 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=162}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-11T03:18:00,210 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=162}] zookeeper.ZKUtil(111): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:18:00,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:18:00,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:18:00,212 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:18:00,213 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=162}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:18:00,213 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=162}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=162 2024-11-11T03:18:00,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=162 2024-11-11T03:18:00,214 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:00,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=162, resume processing ppid=161 2024-11-11T03:18:00,216 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=162, ppid=161, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 261 msec 2024-11-11T03:18:00,218 DEBUG [PEWorker-3 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:18:00,218 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-11T03:18:00,219 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=161, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 272 msec 2024-11-11T03:18:00,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=161 2024-11-11T03:18:00,266 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:18:00,267 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:18:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=163, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:00,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=163 2024-11-11T03:18:00,268 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:18:00,269 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=163, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:00,270 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=163, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-11-11T03:18:00,270 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:00,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=163 2024-11-11T03:18:00,376 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-11T03:18:00,377 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:18:00,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=164, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:00,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=164 2024-11-11T03:18:00,378 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:18:00,379 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=164, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:00,380 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=164, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-11-11T03:18:00,380 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=164 2024-11-11T03:18:00,486 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:18:00,498 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[1] Thread=218 (was 220), OpenFileDescriptor=448 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=489 (was 532), ProcessCount=11 (was 11), AvailableMemoryMB=7272 (was 7297) 2024-11-11T03:18:00,506 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAddRemovePeer[1] Thread=218, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=489, ProcessCount=11, AvailableMemoryMB=7285 2024-11-11T03:18:00,507 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:18:00,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=165, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:18:00,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-11-11T03:18:00,523 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:00,524 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:00,527 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:00,527 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60089 connected 2024-11-11T03:18:00,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-11-11T03:18:00,629 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60089, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:00,634 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:00,785 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:00,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=166 2024-11-11T03:18:00,786 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=166}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-11T03:18:00,809 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=166}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:00,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-11-11T03:18:00,826 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=166}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:18:00,837 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=166}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=166 2024-11-11T03:18:00,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=166 2024-11-11T03:18:00,840 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:00,842 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=166, resume processing ppid=165 2024-11-11T03:18:00,842 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=166, ppid=165, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 206 msec 2024-11-11T03:18:00,842 INFO [PEWorker-5 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-11T03:18:00,844 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=165, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 335 msec 2024-11-11T03:18:00,874 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4696dfd1 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@126a0f6f 2024-11-11T03:18:00,878 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:18:00,879 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:18:00,879 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@5f8b116b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:18:00,880 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:00,887 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:00,887 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:18:00,888 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e6008b connected 2024-11-11T03:18:01,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-11-11T03:18:01,136 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:18:01,137 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:18:01,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=167, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:18:01,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=167 2024-11-11T03:18:01,152 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:01,153 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:01,155 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:01,156 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6008c connected 2024-11-11T03:18:01,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=167 2024-11-11T03:18:01,258 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6008c, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:01,258 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:121) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:18:01,259 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=167, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:121) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:01,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=167, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=122 msec 2024-11-11T03:18:01,260 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:01,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=167 2024-11-11T03:18:01,456 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Replication peer 1 already exists 2024-11-11T03:18:01,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-11-11T03:18:01,457 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:18:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=168, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:01,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=168 2024-11-11T03:18:01,458 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:18:01,459 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=168, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:01,460 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=168, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-11-11T03:18:01,460 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:01,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=168 2024-11-11T03:18:01,566 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:18:01,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-11-11T03:18:01,567 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=2, config=clusterKey=127.0.0.1:54139:/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:18:01,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=169, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:18:01,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=169 2024-11-11T03:18:01,583 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:01,584 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:01,587 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:01,587 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6008d connected 2024-11-11T03:18:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=169 2024-11-11T03:18:01,689 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6008d, quorum=127.0.0.1:54139, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:01,695 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:01,847 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:01,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=170 2024-11-11T03:18:01,847 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=2, type=ADD_PEER 2024-11-11T03:18:01,863 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=170}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:01,877 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.ReplicationSource(229): queueId=2, ReplicationSource: 2, currentBandwidth=0 2024-11-11T03:18:01,881 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=170 2024-11-11T03:18:01,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=170 2024-11-11T03:18:01,883 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 2 for ADD on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:01,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=170, resume processing ppid=169 2024-11-11T03:18:01,884 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=170, ppid=169, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 188 msec 2024-11-11T03:18:01,884 INFO [PEWorker-4 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 2, config clusterKey=127.0.0.1:54139:/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-11T03:18:01,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=169 2024-11-11T03:18:01,886 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=169, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 317 msec 2024-11-11T03:18:01,888 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 2000 millis 2024-11-11T03:18:01,909 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x716ab3e4 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@f37b9c3 2024-11-11T03:18:01,912 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,2 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:18:01,913 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,2 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:18:01,913 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@5fbf5113, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:18:01,914 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 2 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:01,916 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,2-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 20x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:01,916 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,2 {}] regionserver.ReplicationSource(569): peerId=2, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:18:01,916 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 2-0x101955526e6008f connected 2024-11-11T03:18:02,072 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T03:18:02,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=169 2024-11-11T03:18:02,196 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 2 completed 2024-11-11T03:18:02,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-11-11T03:18:02,197 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:18:02,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=171, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:02,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=171 2024-11-11T03:18:02,201 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-11T03:18:02,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-11T03:18:02,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:18:02,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:18:02,204 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=172, ppid=171, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:02,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=171 2024-11-11T03:18:02,355 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:02,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=172 2024-11-11T03:18:02,356 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-11T03:18:02,356 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-11T03:18:02,356 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-11T03:18:02,356 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:18:02,356 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e6008b, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:02,357 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e6008b, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:02,459 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e6008b, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:02,459 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=172}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4696dfd1 to 127.0.0.1:54139 2024-11-11T03:18:02,459 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=172}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:18:02,459 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-11T03:18:02,461 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=172}] zookeeper.ZKUtil(111): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:18:02,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:18:02,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:18:02,464 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:18:02,464 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=172}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:18:02,464 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=172 2024-11-11T03:18:02,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=172 2024-11-11T03:18:02,466 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:02,467 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=172, resume processing ppid=171 2024-11-11T03:18:02,467 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=172, ppid=171, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 263 msec 2024-11-11T03:18:02,472 DEBUG [PEWorker-5 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:18:02,472 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-11T03:18:02,473 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=171, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 275 msec 2024-11-11T03:18:02,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=171 2024-11-11T03:18:02,516 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:18:02,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-11-11T03:18:02,517 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:18:02,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=173, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:02,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=173 2024-11-11T03:18:02,520 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/peer-state 2024-11-11T03:18:02,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/peer-state 2024-11-11T03:18:02,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/2 2024-11-11T03:18:02,522 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2 2024-11-11T03:18:02,523 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=174, ppid=173, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:02,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=173 2024-11-11T03:18:02,674 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:02,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=174 2024-11-11T03:18:02,675 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=174}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=2, type=REMOVE_PEER 2024-11-11T03:18:02,675 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=174}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 2: 0 2024-11-11T03:18:02,675 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=174}] regionserver.ReplicationSource(657): peerId=2, Closing source 2 because: Replication stream was removed by a user 2024-11-11T03:18:02,675 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,2 {}] regionserver.ReplicationSource(508): peerId=2, Interrupted while sleeping between retries 2024-11-11T03:18:02,675 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,2 {}] zookeeper.ZKUtil(165): connection to cluster: 2-0x101955526e6008f, quorum=127.0.0.1:54139, baseZNode=/hbase-test2 Unable to set watcher on znode (/hbase-test2/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:02,675 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,2 {}] zookeeper.ZKWatcher(734): connection to cluster: 2-0x101955526e6008f, quorum=127.0.0.1:54139, baseZNode=/hbase-test2 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:02,777 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,2-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 2-0x101955526e6008f, quorum=127.0.0.1:54139, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:02,777 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=174}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x716ab3e4 to 127.0.0.1:54139 2024-11-11T03:18:02,777 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=174}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:18:02,777 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=174}] regionserver.ReplicationSourceManager(430): Done with the queue 2 2024-11-11T03:18:02,780 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=174}] zookeeper.ZKUtil(111): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/2/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:18:02,781 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/2/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:18:02,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/2 2024-11-11T03:18:02,782 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/2 2024-11-11T03:18:02,782 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=174}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/2 not found in hfile reference queue. 2024-11-11T03:18:02,782 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=174}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=174 2024-11-11T03:18:02,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=174 2024-11-11T03:18:02,784 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 2 for REMOVE on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:02,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=174, resume processing ppid=173 2024-11-11T03:18:02,785 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=174, ppid=173, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 262 msec 2024-11-11T03:18:02,787 DEBUG [PEWorker-3 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/2 not found in hfile reference queue. 2024-11-11T03:18:02,787 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(76): Successfully removed peer 2 2024-11-11T03:18:02,788 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=173, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 270 msec 2024-11-11T03:18:02,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=173 2024-11-11T03:18:02,836 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 2 completed 2024-11-11T03:18:02,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-11-11T03:18:02,836 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:18:02,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=175, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:02,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=175 2024-11-11T03:18:02,838 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:18:02,839 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=175, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:02,839 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=175, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-11-11T03:18:02,840 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=175 2024-11-11T03:18:02,946 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-11T03:18:02,946 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:18:02,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=176, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:02,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=176 2024-11-11T03:18:02,948 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:18:02,948 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=176, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:02,949 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=176, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-11-11T03:18:02,949 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:03,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=176 2024-11-11T03:18:03,057 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:18:03,068 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAddRemovePeer[1] Thread=218 (was 218), OpenFileDescriptor=446 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=489 (was 489), ProcessCount=11 (was 11), AvailableMemoryMB=7267 (was 7285) 2024-11-11T03:18:03,077 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[1] Thread=218, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=489, ProcessCount=11, AvailableMemoryMB=7267 2024-11-11T03:18:03,078 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=whatever,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:18:03,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=177, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:18:03,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=177 2024-11-11T03:18:03,079 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:291) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:288) ~[classes/:?] ... 12 more 2024-11-11T03:18:03,080 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=177, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:291) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:288) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:03,081 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=177, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=2 msec 2024-11-11T03:18:03,081 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:03,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=177 2024-11-11T03:18:03,186 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Can not instantiate configured replication endpoint class=whatever 2024-11-11T03:18:03,187 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:18:03,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=178, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:03,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=178 2024-11-11T03:18:03,188 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:18:03,189 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=178, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:03,189 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=178, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-11-11T03:18:03,189 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:03,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=178 2024-11-11T03:18:03,296 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-11T03:18:03,296 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:18:03,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=179, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:03,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=179 2024-11-11T03:18:03,298 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:18:03,299 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=179, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:03,299 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=179, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-11-11T03:18:03,299 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:03,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=179 2024-11-11T03:18:03,406 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:18:03,417 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[1] Thread=218 (was 218), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=489 (was 489), ProcessCount=11 (was 11), AvailableMemoryMB=7267 (was 7267) 2024-11-11T03:18:03,425 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[1] Thread=218, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=489, ProcessCount=11, AvailableMemoryMB=7267 2024-11-11T03:18:03,426 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=whatever,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:18:03,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=180, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:18:03,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=180 2024-11-11T03:18:03,427 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] ... 13 more 2024-11-11T03:18:03,428 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=180, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:03,429 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=180, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=2 msec 2024-11-11T03:18:03,429 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:03,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=180 2024-11-11T03:18:03,536 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Invalid cluster key: whatever 2024-11-11T03:18:03,537 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:18:03,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=181, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:03,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=181 2024-11-11T03:18:03,539 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:18:03,539 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=181, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:03,540 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=181, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-11-11T03:18:03,540 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:03,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=181 2024-11-11T03:18:03,646 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-11T03:18:03,646 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:18:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=182, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:03,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=182 2024-11-11T03:18:03,648 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:18:03,648 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=182, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:03,649 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=182, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-11-11T03:18:03,649 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:03,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=182 2024-11-11T03:18:03,757 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:18:03,768 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[1] Thread=218 (was 218), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=489 (was 489), ProcessCount=11 (was 11), AvailableMemoryMB=7278 (was 7267) - AvailableMemoryMB LEAK? - 2024-11-11T03:18:03,776 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[1] Thread=218, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=489, ProcessCount=11, AvailableMemoryMB=7278 2024-11-11T03:18:03,777 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:18:03,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=183, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:18:03,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=183 2024-11-11T03:18:03,792 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:03,793 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:03,795 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:03,796 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60090 connected 2024-11-11T03:18:03,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=183 2024-11-11T03:18:03,897 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60090, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:03,903 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=184, ppid=183, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:04,054 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:04,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=184 2024-11-11T03:18:04,055 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-11T03:18:04,070 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=184}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:04,083 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:18:04,094 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=184 2024-11-11T03:18:04,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=184 2024-11-11T03:18:04,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=183 2024-11-11T03:18:04,096 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:04,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=184, resume processing ppid=183 2024-11-11T03:18:04,098 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=184, ppid=183, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 194 msec 2024-11-11T03:18:04,098 INFO [PEWorker-4 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-11T03:18:04,099 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=183, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 321 msec 2024-11-11T03:18:04,127 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x57fe7b2f to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@743c866e 2024-11-11T03:18:04,131 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:18:04,131 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:18:04,131 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@7118a711, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:18:04,132 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:04,134 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:04,135 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:18:04,135 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60092 connected 2024-11-11T03:18:04,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=183 2024-11-11T03:18:04,406 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:18:04,406 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-11T03:18:04,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=185, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:18:04,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=185 2024-11-11T03:18:04,422 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:04,422 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:04,425 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:04,425 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60093 connected 2024-11-11T03:18:04,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=185 2024-11-11T03:18:04,526 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60093, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:04,530 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=186, ppid=185, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:04,681 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:04,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=186 2024-11-11T03:18:04,682 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:18:04,683 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:18:04,683 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:18:04,683 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:18:04,683 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60092, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:04,684 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60092, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:04,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=185 2024-11-11T03:18:04,789 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60092, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:04,789 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=186}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x57fe7b2f to 127.0.0.1:54139 2024-11-11T03:18:04,789 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=186}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:18:04,803 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:18:04,803 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:18:04,803 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=186 2024-11-11T03:18:04,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=186 2024-11-11T03:18:04,805 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:04,806 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=186, resume processing ppid=185 2024-11-11T03:18:04,806 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=186, ppid=185, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 275 msec 2024-11-11T03:18:04,806 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-11T03:18:04,808 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=185, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 400 msec 2024-11-11T03:18:04,837 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x658021db to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3aa112b4 2024-11-11T03:18:04,839 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:18:04,840 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:18:04,840 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@68a08102, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:18:04,841 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:04,843 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:04,843 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:18:04,843 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60095 connected 2024-11-11T03:18:05,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=185 2024-11-11T03:18:05,036 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:18:05,037 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:18:05,037 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-11-11T03:18:05,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=187, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:18:05,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=187 2024-11-11T03:18:05,053 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:05,054 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:05,056 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:05,056 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60096 connected 2024-11-11T03:18:05,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=187 2024-11-11T03:18:05,158 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60096, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:05,160 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=188, ppid=187, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:05,312 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:05,312 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=188 2024-11-11T03:18:05,312 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:18:05,313 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:18:05,313 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:18:05,313 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:18:05,314 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60095, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:05,314 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60095, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:05,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=187 2024-11-11T03:18:05,415 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60095, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:05,415 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=188}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x658021db to 127.0.0.1:54139 2024-11-11T03:18:05,416 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=188}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:18:05,432 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:18:05,433 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:18:05,433 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=188 2024-11-11T03:18:05,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=188 2024-11-11T03:18:05,435 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:05,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=188, resume processing ppid=187 2024-11-11T03:18:05,437 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=188, ppid=187, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 275 msec 2024-11-11T03:18:05,437 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-11-11T03:18:05,438 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=187, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 400 msec 2024-11-11T03:18:05,464 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3feb72eb to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@764e8b4a 2024-11-11T03:18:05,467 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:18:05,468 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:18:05,468 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@2f6548d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:18:05,468 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:05,470 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:05,471 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:18:05,471 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e60098 connected 2024-11-11T03:18:05,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=187 2024-11-11T03:18:05,666 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:18:05,666 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:18:05,667 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:18:05,667 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-11-11T03:18:05,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=189, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:18:05,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=189 2024-11-11T03:18:05,684 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:05,684 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:05,687 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:05,687 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60099 connected 2024-11-11T03:18:05,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=189 2024-11-11T03:18:05,789 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e60099, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:05,792 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=190, ppid=189, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:05,943 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:05,944 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=190 2024-11-11T03:18:05,944 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:18:05,945 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:18:05,945 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:18:05,945 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:18:05,945 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e60098, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:05,945 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e60098, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:05,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=189 2024-11-11T03:18:06,047 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e60098, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:06,047 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=190}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3feb72eb to 127.0.0.1:54139 2024-11-11T03:18:06,047 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=190}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:18:06,061 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:18:06,062 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:18:06,062 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=190 2024-11-11T03:18:06,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=190 2024-11-11T03:18:06,064 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:06,065 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=190, resume processing ppid=189 2024-11-11T03:18:06,065 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-11-11T03:18:06,065 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=190, ppid=189, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 272 msec 2024-11-11T03:18:06,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=189, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 398 msec 2024-11-11T03:18:06,090 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x41181f26 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2deb0fd8 2024-11-11T03:18:06,093 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:18:06,093 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:18:06,094 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3ddf66ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:18:06,094 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:06,097 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:06,097 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:18:06,097 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e6009b connected 2024-11-11T03:18:06,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=189 2024-11-11T03:18:06,296 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:18:06,297 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:18:06,297 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:18:06,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=191, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:06,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=191 2024-11-11T03:18:06,300 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-11T03:18:06,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-11T03:18:06,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:18:06,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:18:06,303 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=192, ppid=191, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:06,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=191 2024-11-11T03:18:06,454 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:06,455 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=192 2024-11-11T03:18:06,455 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=192}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-11T03:18:06,455 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=192}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-11T03:18:06,455 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=192}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-11T03:18:06,455 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:18:06,455 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e6009b, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:06,456 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e6009b, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:06,557 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e6009b, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:06,558 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=192}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x41181f26 to 127.0.0.1:54139 2024-11-11T03:18:06,558 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=192}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:18:06,558 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=192}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-11T03:18:06,560 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=192}] zookeeper.ZKUtil(111): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:18:06,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:18:06,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:18:06,562 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:18:06,562 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=192}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:18:06,562 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=192}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=192 2024-11-11T03:18:06,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=192 2024-11-11T03:18:06,564 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:06,565 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=192, resume processing ppid=191 2024-11-11T03:18:06,565 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=192, ppid=191, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 261 msec 2024-11-11T03:18:06,567 DEBUG [PEWorker-2 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:18:06,567 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-11T03:18:06,568 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=191, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 270 msec 2024-11-11T03:18:06,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=191 2024-11-11T03:18:06,616 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:18:06,616 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:18:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=193, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:06,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=193 2024-11-11T03:18:06,618 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:18:06,618 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=193, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:06,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=193, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-11-11T03:18:06,619 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:06,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=193 2024-11-11T03:18:06,726 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-11T03:18:06,726 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:18:06,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=194, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:06,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=194 2024-11-11T03:18:06,728 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:18:06,728 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=194, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:06,729 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=194, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-11-11T03:18:06,729 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:06,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=194 2024-11-11T03:18:06,836 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:18:06,847 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[1] Thread=218 (was 218), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=450 (was 489), ProcessCount=11 (was 11), AvailableMemoryMB=7271 (was 7278) 2024-11-11T03:18:06,855 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[1] Thread=218, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=450, ProcessCount=11, AvailableMemoryMB=7271 2024-11-11T03:18:06,856 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:18:06,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=195, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:18:06,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=195 2024-11-11T03:18:06,872 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:06,872 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:06,875 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:06,875 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6009c connected 2024-11-11T03:18:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=195 2024-11-11T03:18:06,977 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6009c, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:06,982 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=196, ppid=195, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:07,134 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:07,134 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=196 2024-11-11T03:18:07,134 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-11T03:18:07,149 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=196}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:07,163 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:18:07,169 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=196 2024-11-11T03:18:07,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=196 2024-11-11T03:18:07,171 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:07,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=196, resume processing ppid=195 2024-11-11T03:18:07,173 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=196, ppid=195, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 189 msec 2024-11-11T03:18:07,173 INFO [PEWorker-1 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-11T03:18:07,174 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=195, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 317 msec 2024-11-11T03:18:07,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=195 2024-11-11T03:18:07,176 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:18:07,176 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-11T03:18:07,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=197, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:18:07,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=197 2024-11-11T03:18:07,190 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:07,191 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:07,193 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:07,194 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6009d connected 2024-11-11T03:18:07,198 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3b382095 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@35592027 2024-11-11T03:18:07,201 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:18:07,201 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:18:07,201 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@7a59f72b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:18:07,202 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:07,204 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:07,205 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:18:07,205 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e6009f connected 2024-11-11T03:18:07,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=197 2024-11-11T03:18:07,295 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e6009d, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:07,298 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=198, ppid=197, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:07,450 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:07,450 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=198 2024-11-11T03:18:07,450 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:18:07,451 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:18:07,451 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:18:07,451 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:18:07,451 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e6009f, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:07,452 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e6009f, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=197 2024-11-11T03:18:07,554 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e6009f, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:07,554 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=198}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3b382095 to 127.0.0.1:54139 2024-11-11T03:18:07,554 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=198}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:18:07,568 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:18:07,569 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:18:07,569 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=198 2024-11-11T03:18:07,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=198 2024-11-11T03:18:07,571 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:07,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=198, resume processing ppid=197 2024-11-11T03:18:07,572 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-11T03:18:07,572 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=198, ppid=197, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 273 msec 2024-11-11T03:18:07,573 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=197, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 396 msec 2024-11-11T03:18:07,596 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x71dc2208 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5eb99e35 2024-11-11T03:18:07,600 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:18:07,600 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:18:07,600 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@7af12b9e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:18:07,601 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:07,607 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:07,607 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:18:07,607 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e600a1 connected 2024-11-11T03:18:07,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=197 2024-11-11T03:18:07,806 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:18:07,806 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:18:07,807 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-11-11T03:18:07,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=199, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:18:07,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=199 2024-11-11T03:18:07,822 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:07,823 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:07,826 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:07,826 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e600a2 connected 2024-11-11T03:18:07,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=199 2024-11-11T03:18:07,933 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e600a2, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:07,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=200, ppid=199, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:07,961 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T03:18:08,092 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:08,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=200 2024-11-11T03:18:08,093 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:18:08,094 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:18:08,094 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:18:08,095 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:18:08,095 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e600a1, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:08,095 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e600a1, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:08,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=199 2024-11-11T03:18:08,197 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e600a1, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:08,198 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=200}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x71dc2208 to 127.0.0.1:54139 2024-11-11T03:18:08,198 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=200}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:18:08,214 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:18:08,214 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:18:08,214 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=200 2024-11-11T03:18:08,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=200 2024-11-11T03:18:08,217 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:08,218 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=200, resume processing ppid=199 2024-11-11T03:18:08,218 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=200, ppid=199, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 276 msec 2024-11-11T03:18:08,218 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-11-11T03:18:08,221 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=199, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 412 msec 2024-11-11T03:18:08,251 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2e333a86 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1aece9c8 2024-11-11T03:18:08,260 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:18:08,260 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:18:08,260 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@60694f04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:18:08,261 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:08,264 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:08,264 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:18:08,264 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e600a4 connected 2024-11-11T03:18:08,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=199 2024-11-11T03:18:08,436 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:18:08,436 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:18:08,437 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:18:08,438 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-11-11T03:18:08,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=201, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:18:08,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=201 2024-11-11T03:18:08,462 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:08,463 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:08,465 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:08,466 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e600a5 connected 2024-11-11T03:18:08,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=201 2024-11-11T03:18:08,567 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e600a5, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:08,572 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=202, ppid=201, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:08,724 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:08,725 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=202 2024-11-11T03:18:08,725 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:18:08,727 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:18:08,727 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:18:08,727 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:18:08,727 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e600a4, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:08,728 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e600a4, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:08,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=201 2024-11-11T03:18:08,831 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e600a4, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:08,831 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=202}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2e333a86 to 127.0.0.1:54139 2024-11-11T03:18:08,831 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=202}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:18:08,851 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:18:08,851 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:18:08,852 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=202 2024-11-11T03:18:08,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=202 2024-11-11T03:18:08,854 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:08,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=202, resume processing ppid=201 2024-11-11T03:18:08,857 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=202, ppid=201, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 282 msec 2024-11-11T03:18:08,857 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-11-11T03:18:08,858 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=201, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 419 msec 2024-11-11T03:18:08,891 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x48b1ed0e to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4109200f 2024-11-11T03:18:08,912 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:18:08,912 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:18:08,913 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@1470354e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:18:08,914 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:08,931 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:08,932 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:18:08,948 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e600a7 connected 2024-11-11T03:18:09,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=201 2024-11-11T03:18:09,066 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:18:09,066 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:18:09,068 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:18:09,069 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-11-11T03:18:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=203, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:18:09,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=203 2024-11-11T03:18:09,093 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:09,093 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:09,096 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:09,096 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e600a8 connected 2024-11-11T03:18:09,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=203 2024-11-11T03:18:09,199 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e600a8, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:09,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=204, ppid=203, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:09,354 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:09,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=204 2024-11-11T03:18:09,355 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:18:09,356 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:18:09,356 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:18:09,356 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:18:09,356 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e600a7, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:09,357 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e600a7, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=203 2024-11-11T03:18:09,460 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e600a7, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:09,460 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=204}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x48b1ed0e to 127.0.0.1:54139 2024-11-11T03:18:09,460 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=204}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:18:09,477 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:18:09,477 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:18:09,477 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=204 2024-11-11T03:18:09,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=204 2024-11-11T03:18:09,479 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:09,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=204, resume processing ppid=203 2024-11-11T03:18:09,481 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-11-11T03:18:09,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=204, ppid=203, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 277 msec 2024-11-11T03:18:09,482 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=203, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 412 msec 2024-11-11T03:18:09,519 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x124a0106 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7349b661 2024-11-11T03:18:09,523 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:18:09,523 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:18:09,523 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@ab3e508, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:18:09,525 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:09,528 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:09,528 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:18:09,529 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e600aa connected 2024-11-11T03:18:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=203 2024-11-11T03:18:09,697 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:18:09,697 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:18:09,698 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:18:09,699 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-11-11T03:18:09,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=205, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:18:09,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=205 2024-11-11T03:18:09,720 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:09,721 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:09,726 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:09,728 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e600ab connected 2024-11-11T03:18:09,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=205 2024-11-11T03:18:09,835 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e600ab, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:09,841 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=206, ppid=205, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:09,993 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:09,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=206 2024-11-11T03:18:09,994 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:18:09,995 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:18:09,995 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:18:09,995 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:18:09,995 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e600aa, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:09,996 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e600aa, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:10,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=205 2024-11-11T03:18:10,108 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e600aa, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:10,108 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=206}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x124a0106 to 127.0.0.1:54139 2024-11-11T03:18:10,108 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=206}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:18:10,123 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:18:10,124 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:18:10,124 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=206 2024-11-11T03:18:10,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=206 2024-11-11T03:18:10,126 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:10,128 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=206, resume processing ppid=205 2024-11-11T03:18:10,128 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=206, ppid=205, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 285 msec 2024-11-11T03:18:10,128 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-11-11T03:18:10,130 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=205, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 428 msec 2024-11-11T03:18:10,157 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x688abe1e to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@d82680b 2024-11-11T03:18:10,160 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:18:10,160 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:18:10,161 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@2cd8267a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:18:10,161 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:10,172 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:10,172 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:18:10,173 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e600ad connected 2024-11-11T03:18:10,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=205 2024-11-11T03:18:10,336 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:18:10,336 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:18:10,337 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:18:10,338 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-11-11T03:18:10,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=207, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:18:10,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=207 2024-11-11T03:18:10,360 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:10,361 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:10,364 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:10,364 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e600ae connected 2024-11-11T03:18:10,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=207 2024-11-11T03:18:10,466 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e600ae, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:10,470 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=208, ppid=207, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:10,622 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:10,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=208 2024-11-11T03:18:10,623 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:18:10,624 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:18:10,624 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:18:10,624 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:18:10,624 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e600ad, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:10,624 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e600ad, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:10,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=207 2024-11-11T03:18:10,726 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e600ad, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:10,726 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=208}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x688abe1e to 127.0.0.1:54139 2024-11-11T03:18:10,726 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=208}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:18:10,743 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:18:10,744 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:18:10,744 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=208 2024-11-11T03:18:10,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=208 2024-11-11T03:18:10,746 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:10,748 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=208, resume processing ppid=207 2024-11-11T03:18:10,748 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=208, ppid=207, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 276 msec 2024-11-11T03:18:10,748 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-11-11T03:18:10,749 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=207, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 410 msec 2024-11-11T03:18:10,782 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2e0c979e to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@281c9f5e 2024-11-11T03:18:10,786 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:18:10,786 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:18:10,787 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@2d741f3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:18:10,788 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:10,790 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:10,791 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:18:10,791 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e600b0 connected 2024-11-11T03:18:10,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=207 2024-11-11T03:18:10,966 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:18:10,966 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:18:10,967 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-11-11T03:18:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=209, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:18:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=209 2024-11-11T03:18:10,987 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:10,988 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:10,993 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:10,993 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e600b1 connected 2024-11-11T03:18:11,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=209 2024-11-11T03:18:11,095 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e600b1, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:11,098 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=210, ppid=209, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:11,250 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:11,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=210 2024-11-11T03:18:11,251 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:18:11,252 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=210 2024-11-11T03:18:11,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=210 2024-11-11T03:18:11,255 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:11,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=210, resume processing ppid=209 2024-11-11T03:18:11,257 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=210, ppid=209, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 157 msec 2024-11-11T03:18:11,257 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-11-11T03:18:11,258 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=209, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 290 msec 2024-11-11T03:18:11,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=209 2024-11-11T03:18:11,286 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:18:11,286 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:18:11,287 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:18:11,288 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-11-11T03:18:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=211, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:18:11,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=211 2024-11-11T03:18:11,306 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:11,307 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:11,309 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:11,309 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e600b2 connected 2024-11-11T03:18:11,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=211 2024-11-11T03:18:11,411 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e600b2, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:11,416 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=212, ppid=211, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:11,568 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:11,568 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=212 2024-11-11T03:18:11,569 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:18:11,570 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:18:11,570 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:18:11,570 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:18:11,570 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e600b0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:11,570 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e600b0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:11,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=211 2024-11-11T03:18:11,619 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-11T03:18:11,672 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e600b0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:11,672 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=212}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2e0c979e to 127.0.0.1:54139 2024-11-11T03:18:11,672 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=212}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:18:11,692 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:18:11,692 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:18:11,692 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=212 2024-11-11T03:18:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=212 2024-11-11T03:18:11,694 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:11,697 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=212, resume processing ppid=211 2024-11-11T03:18:11,698 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-11-11T03:18:11,698 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=212, ppid=211, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 278 msec 2024-11-11T03:18:11,699 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=211, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 410 msec 2024-11-11T03:18:11,726 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x30571423 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2c7a57e8 2024-11-11T03:18:11,729 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:18:11,730 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:18:11,730 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@5268bff8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:18:11,731 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:11,733 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:11,734 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:18:11,734 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e600b4 connected 2024-11-11T03:18:11,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=211 2024-11-11T03:18:11,916 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:18:11,916 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:18:11,917 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-11-11T03:18:11,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=213, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:18:11,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=213 2024-11-11T03:18:11,934 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:11,935 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:11,938 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:11,939 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e600b5 connected 2024-11-11T03:18:12,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=213 2024-11-11T03:18:12,041 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e600b5, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:12,044 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=214, ppid=213, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:12,196 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:12,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=214 2024-11-11T03:18:12,197 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:18:12,198 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:18:12,198 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:18:12,198 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:18:12,198 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e600b4, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:12,199 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e600b4, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:12,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=213 2024-11-11T03:18:12,300 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e600b4, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:12,301 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=214}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x30571423 to 127.0.0.1:54139 2024-11-11T03:18:12,301 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=214}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:18:12,322 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:18:12,322 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:18:12,323 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=214 2024-11-11T03:18:12,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=214 2024-11-11T03:18:12,325 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:12,327 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=214, resume processing ppid=213 2024-11-11T03:18:12,327 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=214, ppid=213, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 281 msec 2024-11-11T03:18:12,327 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-11-11T03:18:12,328 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=213, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 410 msec 2024-11-11T03:18:12,358 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3753736b to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@707634e4 2024-11-11T03:18:12,361 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:18:12,362 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:18:12,362 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@16c91777, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:18:12,363 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:12,365 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:12,366 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:18:12,368 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e600b7 connected 2024-11-11T03:18:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=213 2024-11-11T03:18:12,546 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:18:12,546 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:18:12,547 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:18:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=215, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=215 2024-11-11T03:18:12,550 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-11T03:18:12,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-11T03:18:12,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:18:12,553 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:18:12,553 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=216, ppid=215, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=215 2024-11-11T03:18:12,704 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:12,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=216 2024-11-11T03:18:12,705 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=216}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-11T03:18:12,705 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=216}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-11T03:18:12,705 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=216}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-11T03:18:12,705 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:18:12,705 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e600b7, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:12,706 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e600b7, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:12,808 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e600b7, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:12,808 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=216}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3753736b to 127.0.0.1:54139 2024-11-11T03:18:12,808 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=216}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:18:12,808 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=216}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-11T03:18:12,810 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=216}] zookeeper.ZKUtil(111): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:18:12,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:18:12,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:18:12,812 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:18:12,814 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=216}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:18:12,814 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=216}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=216 2024-11-11T03:18:12,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=216 2024-11-11T03:18:12,816 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:12,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=216, resume processing ppid=215 2024-11-11T03:18:12,818 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=216, ppid=215, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 263 msec 2024-11-11T03:18:12,820 DEBUG [PEWorker-5 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:18:12,820 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-11T03:18:12,821 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=215, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 273 msec 2024-11-11T03:18:12,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=215 2024-11-11T03:18:12,866 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:18:12,867 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:18:12,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=217, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:12,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=217 2024-11-11T03:18:12,869 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:18:12,871 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=217, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:12,872 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=217, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-11-11T03:18:12,872 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:12,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=217 2024-11-11T03:18:12,976 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-11T03:18:12,977 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:18:12,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=218, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:12,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=218 2024-11-11T03:18:12,979 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:18:12,979 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=218, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:12,980 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=218, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-11-11T03:18:12,980 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:13,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=218 2024-11-11T03:18:13,086 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:18:13,100 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[1] Thread=213 (was 218), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=446 (was 450), ProcessCount=11 (was 11), AvailableMemoryMB=7232 (was 7271) 2024-11-11T03:18:13,111 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[1] Thread=213, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=446, ProcessCount=11, AvailableMemoryMB=7231 2024-11-11T03:18:13,112 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:18:13,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=219, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:18:13,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=219 2024-11-11T03:18:13,131 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:13,132 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:13,135 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:13,135 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e600b8 connected 2024-11-11T03:18:13,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=219 2024-11-11T03:18:13,237 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e600b8, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:13,244 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=220, ppid=219, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:13,396 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:13,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=220 2024-11-11T03:18:13,396 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-11T03:18:13,415 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=220}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:13,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=219 2024-11-11T03:18:13,431 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:18:13,440 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=220 2024-11-11T03:18:13,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=220 2024-11-11T03:18:13,442 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:13,447 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=220, resume processing ppid=219 2024-11-11T03:18:13,447 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=220, ppid=219, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 198 msec 2024-11-11T03:18:13,447 INFO [PEWorker-3 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-11T03:18:13,448 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=219, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 335 msec 2024-11-11T03:18:13,477 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x57a27316 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7fbd6d7c 2024-11-11T03:18:13,480 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:18:13,480 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:18:13,480 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@1ed0e132, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:18:13,481 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:13,483 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:13,484 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:18:13,484 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e600ba connected 2024-11-11T03:18:13,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=219 2024-11-11T03:18:13,736 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:18:13,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-11-11T03:18:13,737 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3938): Client=jenkins//172.17.0.2 disable replication peer, id=1 2024-11-11T03:18:13,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=221, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure 2024-11-11T03:18:13,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=221 2024-11-11T03:18:13,741 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=222, ppid=221, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:13,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=221 2024-11-11T03:18:13,893 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:13,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=222 2024-11-11T03:18:13,894 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=DISABLE_PEER 2024-11-11T03:18:13,894 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-11T03:18:13,894 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-11T03:18:13,895 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:18:13,895 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e600ba, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:13,895 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e600ba, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:13,997 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e600ba, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:13,997 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=222}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x57a27316 to 127.0.0.1:54139 2024-11-11T03:18:13,997 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=222}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:18:14,016 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:18:14,017 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-11T03:18:14,017 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=222 2024-11-11T03:18:14,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=222 2024-11-11T03:18:14,019 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for DISABLE on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:14,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=222, resume processing ppid=221 2024-11-11T03:18:14,021 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=222, ppid=221, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 278 msec 2024-11-11T03:18:14,021 INFO [PEWorker-1 {}] replication.DisablePeerProcedure(67): Successfully disabled peer 1 2024-11-11T03:18:14,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=221, state=SUCCESS; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure in 284 msec 2024-11-11T03:18:14,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=221 2024-11-11T03:18:14,056 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: DISABLE_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:18:14,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-11-11T03:18:14,058 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:18:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=223, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=223 2024-11-11T03:18:14,060 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4d189133 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7e8e2fa4 2024-11-11T03:18:14,061 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-11T03:18:14,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-11T03:18:14,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:18:14,064 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:18:14,065 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=224, ppid=223, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:14,073 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:18:14,073 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:18:14,073 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@de08a57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:18:14,074 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:14,084 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:14,084 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:18:14,085 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e600bc connected 2024-11-11T03:18:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=223 2024-11-11T03:18:14,217 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:14,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=224 2024-11-11T03:18:14,218 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=224}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-11T03:18:14,218 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=224}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-11T03:18:14,218 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=224}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-11T03:18:14,218 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:18:14,218 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e600bc, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:14,218 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e600bc, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:14,320 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e600bc, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:14,320 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=224}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4d189133 to 127.0.0.1:54139 2024-11-11T03:18:14,320 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=224}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:18:14,321 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=224}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-11T03:18:14,322 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=224}] zookeeper.ZKUtil(111): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:18:14,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:18:14,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:18:14,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:18:14,325 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=224}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:18:14,325 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=224}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=224 2024-11-11T03:18:14,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=224 2024-11-11T03:18:14,327 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:14,329 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=224, resume processing ppid=223 2024-11-11T03:18:14,329 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=224, ppid=223, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 262 msec 2024-11-11T03:18:14,332 DEBUG [PEWorker-5 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:18:14,332 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-11T03:18:14,333 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=223, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 274 msec 2024-11-11T03:18:14,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=223 2024-11-11T03:18:14,376 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:18:14,377 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:18:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=225, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:14,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=225 2024-11-11T03:18:14,379 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:18:14,380 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=225, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:14,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=225, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-11-11T03:18:14,380 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:14,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=225 2024-11-11T03:18:14,487 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-11T03:18:14,487 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:18:14,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=226, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:14,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=226 2024-11-11T03:18:14,489 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:18:14,490 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=226, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:14,491 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=226, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-11-11T03:18:14,491 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:14,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=226 2024-11-11T03:18:14,596 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:18:14,610 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[1] Thread=213 (was 213), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=446 (was 446), ProcessCount=11 (was 11), AvailableMemoryMB=7227 (was 7231) 2024-11-11T03:18:14,620 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[1] Thread=213, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=446, ProcessCount=11, AvailableMemoryMB=7227 2024-11-11T03:18:14,621 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=dummy_peer 2024-11-11T03:18:14,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=227, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:14,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=227 2024-11-11T03:18:14,624 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer dummy_peer, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:18:14,625 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=227, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:14,626 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=227, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-11-11T03:18:14,626 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=227 2024-11-11T03:18:14,726 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: dummy_peer failed with dummy_peer 2024-11-11T03:18:14,727 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:18:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=228, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:14,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=228 2024-11-11T03:18:14,729 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:18:14,730 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=228, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:14,730 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=228, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-11-11T03:18:14,730 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:14,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=228 2024-11-11T03:18:14,836 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-11T03:18:14,837 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:18:14,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=229, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:14,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=229 2024-11-11T03:18:14,838 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:18:14,839 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=229, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:14,840 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=229, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-11-11T03:18:14,840 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:14,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=229 2024-11-11T03:18:14,946 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:18:14,961 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[1] Thread=213 (was 213), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=434 (was 446), ProcessCount=11 (was 11), AvailableMemoryMB=7227 (was 7227) 2024-11-11T03:18:14,971 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerConfig[1] Thread=213, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=434, ProcessCount=11, AvailableMemoryMB=7226 2024-11-11T03:18:14,972 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:18:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=230, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:18:14,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=230 2024-11-11T03:18:14,991 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:14,992 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:14,996 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:14,997 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e600bd connected 2024-11-11T03:18:15,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=230 2024-11-11T03:18:15,099 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e600bd, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:15,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=231, ppid=230, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:15,255 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:15,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=231 2024-11-11T03:18:15,256 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-11T03:18:15,278 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=231}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:15,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=230 2024-11-11T03:18:15,294 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:18:15,301 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=231 2024-11-11T03:18:15,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=231 2024-11-11T03:18:15,303 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:15,305 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=231, resume processing ppid=230 2024-11-11T03:18:15,305 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=231, ppid=230, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 199 msec 2024-11-11T03:18:15,305 INFO [PEWorker-4 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-11T03:18:15,307 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=230, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 333 msec 2024-11-11T03:18:15,335 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2ac5f78a to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@975a9b5 2024-11-11T03:18:15,338 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:18:15,339 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:18:15,339 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@5657e61e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:18:15,341 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:15,344 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:15,344 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:18:15,344 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e600bf connected 2024-11-11T03:18:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=230 2024-11-11T03:18:15,596 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:18:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-11-11T03:18:15,597 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:18:15,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=232, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:15,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=232 2024-11-11T03:18:15,600 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-11T03:18:15,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-11T03:18:15,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:18:15,602 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:18:15,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=233, ppid=232, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:15,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=232 2024-11-11T03:18:15,755 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:15,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=233 2024-11-11T03:18:15,756 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=233}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-11T03:18:15,756 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=233}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-11T03:18:15,756 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=233}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-11T03:18:15,756 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:18:15,756 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e600bf, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:15,756 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e600bf, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:15,859 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e600bf, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:15,859 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=233}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2ac5f78a to 127.0.0.1:54139 2024-11-11T03:18:15,859 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=233}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:18:15,859 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=233}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-11T03:18:15,861 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=233}] zookeeper.ZKUtil(111): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:18:15,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:18:15,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:18:15,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:18:15,864 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=233}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:18:15,865 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=233}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=233 2024-11-11T03:18:15,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=233 2024-11-11T03:18:15,866 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:15,868 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=233, resume processing ppid=232 2024-11-11T03:18:15,868 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=233, ppid=232, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 263 msec 2024-11-11T03:18:15,871 DEBUG [PEWorker-2 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:18:15,871 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-11T03:18:15,872 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=232, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 274 msec 2024-11-11T03:18:15,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=232 2024-11-11T03:18:15,916 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:18:15,917 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:18:15,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=234, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:15,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=234 2024-11-11T03:18:15,918 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:18:15,919 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=234, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:15,920 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=234, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-11-11T03:18:15,920 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:16,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=234 2024-11-11T03:18:16,027 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-11T03:18:16,027 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:18:16,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=235, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:16,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=235 2024-11-11T03:18:16,029 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:18:16,030 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=235, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:16,031 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=235, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-11-11T03:18:16,031 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:16,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=235 2024-11-11T03:18:16,137 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:18:16,149 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerConfig[1] Thread=213 (was 213), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=434 (was 434), ProcessCount=11 (was 11), AvailableMemoryMB=7232 (was 7226) - AvailableMemoryMB LEAK? - 2024-11-11T03:18:16,158 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[1] Thread=213, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=434, ProcessCount=11, AvailableMemoryMB=7232 2024-11-11T03:18:16,161 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:18:16,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-11T03:18:16,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] ipc.CallRunner(138): callId: 755 service: MasterService methodName: AddReplicationPeer size: 80 connection: 172.17.0.2:39256 deadline: 1731295156161, exception=java.io.IOException: Replication peer modification disabled 2024-11-11T03:18:16,162 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(168): Call to master failed, tries = 1, maxAttempts = 2, timeout = 120000 ms, time elapsed = 1 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:245) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:431) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 33 more 2024-11-11T03:18:16,266 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:18:16,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-11T03:18:16,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] ipc.CallRunner(138): callId: 756 service: MasterService methodName: AddReplicationPeer size: 80 connection: 172.17.0.2:39256 deadline: 1731295156266, exception=java.io.IOException: Replication peer modification disabled 2024-11-11T03:18:16,267 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(168): Call to master failed, tries = 2, maxAttempts = 2, timeout = 120000 ms, time elapsed = 106 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:245) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:431) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 33 more 2024-11-11T03:18:16,268 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Failed after attempts=2, exceptions: 2024-11-11T03:18:16.163Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) 2024-11-11T03:18:16.268Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) 2024-11-11T03:18:16,270 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:18:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=236, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:16,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=236 2024-11-11T03:18:16,272 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:18:16,273 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=236, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:16,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=236, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-11-11T03:18:16,274 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:16,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=236 2024-11-11T03:18:16,377 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-11T03:18:16,377 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:18:16,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=237, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:16,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=237 2024-11-11T03:18:16,379 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:18:16,381 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=237, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:16,382 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=237, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-11-11T03:18:16,382 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:16,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=237 2024-11-11T03:18:16,487 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:18:16,504 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[1] Thread=213 (was 213), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=434 (was 434), ProcessCount=11 (was 11), AvailableMemoryMB=7222 (was 7232) 2024-11-11T03:18:16,516 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[1] Thread=213, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=434, ProcessCount=11, AvailableMemoryMB=7226 2024-11-11T03:18:16,517 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:18:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=238, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:18:16,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=238 2024-11-11T03:18:16,533 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:16,534 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:16,537 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:16,537 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e600c0 connected 2024-11-11T03:18:16,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=238 2024-11-11T03:18:16,639 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e600c0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:16,644 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=239, ppid=238, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:16,796 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:16,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=239 2024-11-11T03:18:16,797 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-11T03:18:16,814 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=239}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:16,830 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:18:16,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=238 2024-11-11T03:18:16,837 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=239 2024-11-11T03:18:16,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=239 2024-11-11T03:18:16,839 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:16,840 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=239, resume processing ppid=238 2024-11-11T03:18:16,840 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=239, ppid=238, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 195 msec 2024-11-11T03:18:16,840 INFO [PEWorker-1 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-11T03:18:16,842 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=238, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 324 msec 2024-11-11T03:18:16,871 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x337783a2 to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3a48084f 2024-11-11T03:18:16,883 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:18:16,883 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:18:16,883 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@66a44306, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:18:16,885 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:16,888 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:16,889 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:18:16,896 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e600c2 connected 2024-11-11T03:18:17,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=238 2024-11-11T03:18:17,146 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:18:17,147 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:18:17,147 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:18:17,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=240, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:17,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=240 2024-11-11T03:18:17,151 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-11T03:18:17,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-11T03:18:17,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:18:17,153 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:18:17,153 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=241, ppid=240, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:17,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=240 2024-11-11T03:18:17,305 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:17,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=241 2024-11-11T03:18:17,306 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=241}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-11T03:18:17,306 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=241}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-11T03:18:17,306 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=241}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-11T03:18:17,306 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:18:17,306 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e600c2, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:17,307 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e600c2, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:17,409 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e600c2, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:17,409 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=241}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x337783a2 to 127.0.0.1:54139 2024-11-11T03:18:17,409 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=241}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:18:17,409 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=241}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-11T03:18:17,411 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=241}] zookeeper.ZKUtil(111): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:18:17,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:18:17,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:18:17,414 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:18:17,414 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=241}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:18:17,414 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=241}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=241 2024-11-11T03:18:17,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=241 2024-11-11T03:18:17,416 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:17,418 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=241, resume processing ppid=240 2024-11-11T03:18:17,418 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=241, ppid=240, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 263 msec 2024-11-11T03:18:17,421 DEBUG [PEWorker-5 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:18:17,421 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-11T03:18:17,422 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=240, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 274 msec 2024-11-11T03:18:17,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=240 2024-11-11T03:18:17,466 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:18:17,466 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:18:17,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=242, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:17,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=242 2024-11-11T03:18:17,468 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:18:17,469 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=242, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:17,470 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=242, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-11-11T03:18:17,470 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:17,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=242 2024-11-11T03:18:17,576 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:18:17,590 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[1] Thread=213 (was 213), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=434 (was 434), ProcessCount=11 (was 11), AvailableMemoryMB=7224 (was 7226) 2024-11-11T03:18:17,600 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerBandwidth[1] Thread=213, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=434, ProcessCount=11, AvailableMemoryMB=7224 2024-11-11T03:18:17,600 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:18:17,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=243, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:18:17,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=243 2024-11-11T03:18:17,622 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:17,623 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:17,625 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:17,626 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e600c3 connected 2024-11-11T03:18:17,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=243 2024-11-11T03:18:17,728 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e600c3, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:17,736 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=244, ppid=243, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:17,887 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:17,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=244 2024-11-11T03:18:17,888 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=244}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-11T03:18:17,906 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=244}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:17,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=243 2024-11-11T03:18:17,922 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=244}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:18:17,929 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=244}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=244 2024-11-11T03:18:17,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=244 2024-11-11T03:18:17,932 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:17,934 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=244, resume processing ppid=243 2024-11-11T03:18:17,934 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=244, ppid=243, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 196 msec 2024-11-11T03:18:17,934 INFO [PEWorker-5 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-11T03:18:17,935 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=243, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 334 msec 2024-11-11T03:18:17,974 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4fff070b to 127.0.0.1:54139 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1a478ddd 2024-11-11T03:18:17,978 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-11T03:18:17,979 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-11T03:18:17,979 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3ba843bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-11T03:18:17,980 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:17,982 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:17,982 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-11T03:18:17,983 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x101955526e600c5 connected 2024-11-11T03:18:17,996 DEBUG [master/e4cdd3a0b883:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region cf6c7e153be0acd168cc278b105d726f changed from -1.0 to 0.0, refreshing cache 2024-11-11T03:18:18,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=243 2024-11-11T03:18:18,226 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:18:18,226 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:18:18,227 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-11-11T03:18:18,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=245, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-11T03:18:18,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=245 2024-11-11T03:18:18,250 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-11T03:18:18,250 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:54139 2024-11-11T03:18:18,253 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id0x0, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-11T03:18:18,253 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e600c6 connected 2024-11-11T03:18:18,310 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T03:18:18,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=245 2024-11-11T03:18:18,356 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@8b6fafcheck-peer-cluster-id-0x101955526e600c6, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:18,360 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=246, ppid=245, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:18,512 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:18,512 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=246 2024-11-11T03:18:18,513 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-11T03:18:18,514 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=246 2024-11-11T03:18:18,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=246 2024-11-11T03:18:18,516 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:18,517 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=246, resume processing ppid=245 2024-11-11T03:18:18,517 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=246, ppid=245, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 156 msec 2024-11-11T03:18:18,517 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:54139:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-11-11T03:18:18,518 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=245, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 290 msec 2024-11-11T03:18:18,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=245 2024-11-11T03:18:18,546 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-11T03:18:18,546 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-11T03:18:18,547 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:18:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=247, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:18,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=247 2024-11-11T03:18:18,550 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-11T03:18:18,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-11T03:18:18,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:18:18,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:18:18,553 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=248, ppid=247, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:18,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=247 2024-11-11T03:18:18,704 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:18,705 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=248 2024-11-11T03:18:18,705 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=248}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-11T03:18:18,705 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=248}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-11T03:18:18,705 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=248}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-11T03:18:18,705 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-11T03:18:18,705 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x101955526e600c5, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:18,706 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x101955526e600c5, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-11T03:18:18,808 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x101955526e600c5, quorum=127.0.0.1:54139, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:18,808 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=248}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4fff070b to 127.0.0.1:54139 2024-11-11T03:18:18,808 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=248}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:18:18,808 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=248}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-11T03:18:18,810 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=248}] zookeeper.ZKUtil(111): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:18:18,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:18:18,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:18:18,813 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:18:18,814 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=248}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:18:18,814 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=248}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=248 2024-11-11T03:18:18,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=248 2024-11-11T03:18:18,816 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:18,817 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=248, resume processing ppid=247 2024-11-11T03:18:18,817 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=248, ppid=247, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 263 msec 2024-11-11T03:18:18,820 DEBUG [PEWorker-2 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:18:18,820 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-11T03:18:18,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=247, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 273 msec 2024-11-11T03:18:18,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=247 2024-11-11T03:18:18,866 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:18:18,867 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:18:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=249, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:18,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=249 2024-11-11T03:18:18,869 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:18:18,870 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=249, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:18,870 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=249, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-11-11T03:18:18,870 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:18,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=249 2024-11-11T03:18:18,977 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-11T03:18:18,977 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:18:18,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=250, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=250 2024-11-11T03:18:18,979 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:18:18,979 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=250, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:18,980 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=250, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-11-11T03:18:18,980 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:19,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=250 2024-11-11T03:18:19,087 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:18:19,102 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerBandwidth[1] Thread=216 (was 213) - Thread LEAK? -, OpenFileDescriptor=448 (was 446) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=434 (was 434), ProcessCount=11 (was 11), AvailableMemoryMB=7220 (was 7224) 2024-11-11T03:18:19,127 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[1] Thread=216, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=434, ProcessCount=11, AvailableMemoryMB=7219 2024-11-11T03:18:19,127 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:18:19,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=251, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:18:19,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=251 2024-11-11T03:18:19,137 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=252, ppid=251, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:19,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=251 2024-11-11T03:18:19,289 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:19,289 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=252 2024-11-11T03:18:19,290 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=252}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-11T03:18:19,334 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=252}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-11T03:18:19,343 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=252}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=252 2024-11-11T03:18:19,343 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(587): peerId=1, queueId=1 (queues=1) is replicating from cluster=454cc11f-9833-4fa8-a013-fffc9bade1b4 to cluster=454cc11f-9833-4fa8-a013-fffc9bade1b4 2024-11-11T03:18:19,343 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSource(343): peerId=1, starting shipping worker for walGroupId=e4cdd3a0b883%2C41101%2C1731295033876 2024-11-11T03:18:19,343 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1 {}] regionserver.ReplicationSourceWALReader(112): peerClusterZnode=1, ReplicationSourceWALReaderThread : 1 inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-11T03:18:19,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=252 2024-11-11T03:18:19,344 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1.replicationSource.shippere4cdd3a0b883%2C41101%2C1731295033876,1 {}] regionserver.ReplicationSourceShipper(98): Running ReplicationSourceShipper Thread for wal group: e4cdd3a0b883%2C41101%2C1731295033876 2024-11-11T03:18:19,345 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1.replicationSource.wal-reader.e4cdd3a0b883%2C41101%2C1731295033876,1 {}] regionserver.WALEntryStream(260): Creating new reader hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/WALs/e4cdd3a0b883,41101,1731295033876/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784, startPosition=0, beingWritten=true 2024-11-11T03:18:19,345 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:19,349 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=252, resume processing ppid=251 2024-11-11T03:18:19,349 INFO [PEWorker-4 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-11T03:18:19,349 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=252, ppid=251, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 208 msec 2024-11-11T03:18:19,350 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=251, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 222 msec 2024-11-11T03:18:19,357 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1.replicationSource.wal-reader.e4cdd3a0b883%2C41101%2C1731295033876,1 {}] regionserver.ReplicationSourceWALReader(181): Read 0 WAL entries eligible for replication 2024-11-11T03:18:19,357 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1.replicationSource.wal-reader.e4cdd3a0b883%2C41101%2C1731295033876,1 {}] regionserver.WALEntryStream(223): Reset reader hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/WALs/e4cdd3a0b883,41101,1731295033876/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 to pos 589, reset compression=false 2024-11-11T03:18:19,358 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1.replicationSource.shippere4cdd3a0b883%2C41101%2C1731295033876,1 {}] regionserver.ReplicationSourceShipper(110): Shipper from source 1 got entry batch from reader: WALEntryBatch [walEntries=[], lastWalPath=hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/WALs/e4cdd3a0b883,41101,1731295033876/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784, lastWalPosition=589, nbRowKeys=0, nbHFiles=0, heapSize=0, lastSeqIds={}, endOfFile=false,usedBufferSize=0] 2024-11-11T03:18:19,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=251 2024-11-11T03:18:19,446 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:18:19,447 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=2, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-11T03:18:19,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=253, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-11T03:18:19,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=253 2024-11-11T03:18:19,449 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] ... 13 more 2024-11-11T03:18:19,449 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=253, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:19,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=253, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=3 msec 2024-11-11T03:18:19,450 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:19,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=253 2024-11-11T03:18:19,557 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 2 failed with Invalid cluster key: 2024-11-11T03:18:19,557 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-11T03:18:19,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=254, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:19,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-11-11T03:18:19,561 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-11T03:18:19,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-11T03:18:19,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:18:19,565 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-11T03:18:19,565 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=255, ppid=254, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-11T03:18:19,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-11-11T03:18:19,717 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:19,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=41101 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=255 2024-11-11T03:18:19,718 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=255}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-11T03:18:19,718 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=255}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-11T03:18:19,718 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=255}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-11T03:18:19,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-11-11T03:18:20,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-11-11T03:18:20,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-11-11T03:18:20,718 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1.replicationSource.wal-reader.e4cdd3a0b883%2C41101%2C1731295033876,1 {}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:131) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:166) ~[classes/:?] 2024-11-11T03:18:20,718 WARN [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1.replicationSource.shippere4cdd3a0b883%2C41101%2C1731295033876,1 {}] regionserver.ReplicationSourceShipper(123): Interrupted while waiting for next replication entry batch java.lang.InterruptedException: null at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1681) ~[?:?] at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) ~[?:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.poll(ReplicationSourceWALReader.java:313) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceShipper.run(ReplicationSourceShipper.java:109) ~[classes/:?] 2024-11-11T03:18:21,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-11-11T03:18:21,718 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=255}] regionserver.ReplicationSource(711): peerId=1, ReplicationSourceWorker RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0.replicationSource,1.replicationSource.shippere4cdd3a0b883%2C41101%2C1731295033876,1 terminated 2024-11-11T03:18:21,718 INFO [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=255}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-11T03:18:21,719 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=255}] zookeeper.ZKUtil(111): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:18:21,721 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1/e4cdd3a0b883%2C41101%2C1731295033876.1731295036784 2024-11-11T03:18:21,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:18:21,722 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/e4cdd3a0b883,41101,1731295033876/1 2024-11-11T03:18:21,722 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=255}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:18:21,722 DEBUG [RS_REFRESH_PEER-regionserver/e4cdd3a0b883:0-0 {event_type=RS_REFRESH_PEER, pid=255}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=255 2024-11-11T03:18:21,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(4106): Remote procedure done, pid=255 2024-11-11T03:18:21,724 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on e4cdd3a0b883,41101,1731295033876 suceeded 2024-11-11T03:18:21,725 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=255, resume processing ppid=254 2024-11-11T03:18:21,726 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=255, ppid=254, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 2.1590 sec 2024-11-11T03:18:21,728 DEBUG [PEWorker-4 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-11T03:18:21,728 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-11T03:18:21,729 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=254, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 2.1710 sec 2024-11-11T03:18:23,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-11-11T03:18:23,716 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-11T03:18:23,717 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-11T03:18:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] procedure2.ProcedureExecutor(1098): Stored pid=256, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-11T03:18:23,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=256 2024-11-11T03:18:23,718 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-11T03:18:23,719 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=256, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-11T03:18:23,720 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=256, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-11-11T03:18:23,720 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-11T03:18:23,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36699 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=256 2024-11-11T03:18:23,827 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-11T03:18:23,843 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[1] Thread=213 (was 216), OpenFileDescriptor=444 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=423 (was 434), ProcessCount=11 (was 11), AvailableMemoryMB=7201 (was 7219) 2024-11-11T03:18:23,844 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-11-11T03:18:23,844 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.client.TestAsyncAdminBase.tearDownAfterClass(TestAsyncAdminBase.java:96) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-11T03:18:23,855 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:18:23,856 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4b613db6 to 127.0.0.1:54139 2024-11-11T03:18:23,856 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-11-11T03:18:23,856 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x665ee9b6 to 127.0.0.1:54139 2024-11-11T03:18:23,856 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:18:23,856 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-11T03:18:23,856 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1029449637, stopped=false 2024-11-11T03:18:23,856 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=e4cdd3a0b883,36699,1731295032496 2024-11-11T03:18:23,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T03:18:23,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:18:23,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-11T03:18:23,858 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:18:23,858 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-11-11T03:18:23,859 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T03:18:23,859 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-11T03:18:23,859 INFO [RS:0;e4cdd3a0b883:41101 {}] regionserver.HRegionServer(1136): Closing user regions 2024-11-11T03:18:23,859 INFO [RS:0;e4cdd3a0b883:41101 {}] regionserver.HRegionServer(3579): Received CLOSE for cf6c7e153be0acd168cc278b105d726f 2024-11-11T03:18:23,859 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:18:23,860 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'e4cdd3a0b883,41101,1731295033876' ***** 2024-11-11T03:18:23,860 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-11-11T03:18:23,861 DEBUG [RS_CLOSE_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing cf6c7e153be0acd168cc278b105d726f, disabling compactions & flushes 2024-11-11T03:18:23,861 INFO [RS_CLOSE_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1731295037996.cf6c7e153be0acd168cc278b105d726f. 2024-11-11T03:18:23,861 DEBUG [RS_CLOSE_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1731295037996.cf6c7e153be0acd168cc278b105d726f. 2024-11-11T03:18:23,862 DEBUG [RS_CLOSE_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1731295037996.cf6c7e153be0acd168cc278b105d726f. after waiting 0 ms 2024-11-11T03:18:23,862 DEBUG [RS_CLOSE_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1731295037996.cf6c7e153be0acd168cc278b105d726f. 2024-11-11T03:18:23,862 INFO [RS:0;e4cdd3a0b883:41101 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-11T03:18:23,862 INFO [RS:0;e4cdd3a0b883:41101 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-11T03:18:23,862 INFO [RS:0;e4cdd3a0b883:41101 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-11T03:18:23,862 INFO [RS:0;e4cdd3a0b883:41101 {}] regionserver.HRegionServer(1224): stopping server e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:23,862 DEBUG [RS:0;e4cdd3a0b883:41101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:18:23,862 INFO [RS:0;e4cdd3a0b883:41101 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-11T03:18:23,862 INFO [RS:0;e4cdd3a0b883:41101 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-11T03:18:23,862 INFO [RS:0;e4cdd3a0b883:41101 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-11T03:18:23,862 INFO [RS_CLOSE_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing cf6c7e153be0acd168cc278b105d726f 1/1 column families, dataSize=78 B heapSize=488 B 2024-11-11T03:18:23,862 INFO [RS:0;e4cdd3a0b883:41101 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-11-11T03:18:23,863 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-11-11T03:18:23,864 INFO [RS:0;e4cdd3a0b883:41101 {}] regionserver.HRegionServer(1599): Waiting on 2 regions to close 2024-11-11T03:18:23,864 DEBUG [RS:0;e4cdd3a0b883:41101 {}] regionserver.HRegionServer(1603): Online Regions={cf6c7e153be0acd168cc278b105d726f=hbase:namespace,,1731295037996.cf6c7e153be0acd168cc278b105d726f., 1588230740=hbase:meta,,1.1588230740} 2024-11-11T03:18:23,865 DEBUG [RS:0;e4cdd3a0b883:41101 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, cf6c7e153be0acd168cc278b105d726f 2024-11-11T03:18:23,866 DEBUG [RS_CLOSE_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-11-11T03:18:23,866 INFO [RS_CLOSE_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-11-11T03:18:23,867 DEBUG [RS_CLOSE_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-11-11T03:18:23,867 DEBUG [RS_CLOSE_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-11T03:18:23,867 DEBUG [RS_CLOSE_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-11-11T03:18:23,867 INFO [RS_CLOSE_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=1.23 KB heapSize=2.87 KB 2024-11-11T03:18:23,889 INFO [regionserver/e4cdd3a0b883:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T03:18:23,953 DEBUG [RS_CLOSE_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/data/hbase/meta/1588230740/.tmp/info/56bb81f572834e95806da4d0eae71ab1 is 143, key is hbase:namespace,,1731295037996.cf6c7e153be0acd168cc278b105d726f./info:regioninfo/1731295038620/Put/seqid=0 2024-11-11T03:18:23,953 DEBUG [RS_CLOSE_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/data/hbase/namespace/cf6c7e153be0acd168cc278b105d726f/.tmp/info/5363f18b0c79471696fcd1fde408dce4 is 45, key is default/info:d/1731295038805/Put/seqid=0 2024-11-11T03:18:23,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43237 is added to blk_1073741837_1013 (size=6595) 2024-11-11T03:18:23,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43237 is added to blk_1073741838_1014 (size=5037) 2024-11-11T03:18:23,977 INFO [RS_CLOSE_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/data/hbase/namespace/cf6c7e153be0acd168cc278b105d726f/.tmp/info/5363f18b0c79471696fcd1fde408dce4 2024-11-11T03:18:24,033 DEBUG [RS_CLOSE_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/data/hbase/namespace/cf6c7e153be0acd168cc278b105d726f/.tmp/info/5363f18b0c79471696fcd1fde408dce4 as hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/data/hbase/namespace/cf6c7e153be0acd168cc278b105d726f/info/5363f18b0c79471696fcd1fde408dce4 2024-11-11T03:18:24,050 INFO [RS_CLOSE_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/data/hbase/namespace/cf6c7e153be0acd168cc278b105d726f/info/5363f18b0c79471696fcd1fde408dce4, entries=2, sequenceid=6, filesize=4.9 K 2024-11-11T03:18:24,055 INFO [RS_CLOSE_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for cf6c7e153be0acd168cc278b105d726f in 193ms, sequenceid=6, compaction requested=false 2024-11-11T03:18:24,065 DEBUG [RS:0;e4cdd3a0b883:41101 {}] regionserver.HRegionServer(1629): Waiting on 1588230740, cf6c7e153be0acd168cc278b105d726f 2024-11-11T03:18:24,073 DEBUG [RS_CLOSE_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/data/hbase/namespace/cf6c7e153be0acd168cc278b105d726f/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-11-11T03:18:24,076 INFO [RS_CLOSE_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1731295037996.cf6c7e153be0acd168cc278b105d726f. 2024-11-11T03:18:24,076 DEBUG [RS_CLOSE_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for cf6c7e153be0acd168cc278b105d726f: 2024-11-11T03:18:24,076 DEBUG [RS_CLOSE_REGION-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1731295037996.cf6c7e153be0acd168cc278b105d726f. 2024-11-11T03:18:24,265 DEBUG [RS:0;e4cdd3a0b883:41101 {}] regionserver.HRegionServer(1629): Waiting on 1588230740 2024-11-11T03:18:24,310 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-11T03:18:24,385 INFO [RS_CLOSE_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.14 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/data/hbase/meta/1588230740/.tmp/info/56bb81f572834e95806da4d0eae71ab1 2024-11-11T03:18:24,442 DEBUG [RS_CLOSE_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/data/hbase/meta/1588230740/.tmp/table/0d9c4ab4ec7f470fa73466e279dd4b67 is 51, key is hbase:namespace/table:state/1731295038687/Put/seqid=0 2024-11-11T03:18:24,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43237 is added to blk_1073741839_1015 (size=5242) 2024-11-11T03:18:24,458 INFO [RS_CLOSE_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=94 B at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/data/hbase/meta/1588230740/.tmp/table/0d9c4ab4ec7f470fa73466e279dd4b67 2024-11-11T03:18:24,465 DEBUG [RS:0;e4cdd3a0b883:41101 {}] regionserver.HRegionServer(1629): Waiting on 1588230740 2024-11-11T03:18:24,474 DEBUG [RS_CLOSE_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/data/hbase/meta/1588230740/.tmp/info/56bb81f572834e95806da4d0eae71ab1 as hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/data/hbase/meta/1588230740/info/56bb81f572834e95806da4d0eae71ab1 2024-11-11T03:18:24,489 INFO [RS_CLOSE_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/data/hbase/meta/1588230740/info/56bb81f572834e95806da4d0eae71ab1, entries=10, sequenceid=9, filesize=6.4 K 2024-11-11T03:18:24,492 DEBUG [RS_CLOSE_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/data/hbase/meta/1588230740/.tmp/table/0d9c4ab4ec7f470fa73466e279dd4b67 as hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/data/hbase/meta/1588230740/table/0d9c4ab4ec7f470fa73466e279dd4b67 2024-11-11T03:18:24,504 INFO [RS_CLOSE_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/data/hbase/meta/1588230740/table/0d9c4ab4ec7f470fa73466e279dd4b67, entries=2, sequenceid=9, filesize=5.1 K 2024-11-11T03:18:24,505 INFO [RS_CLOSE_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~1.23 KB/1264, heapSize ~2.59 KB/2648, currentSize=0 B/0 for 1588230740 in 638ms, sequenceid=9, compaction requested=false 2024-11-11T03:18:24,515 DEBUG [RS_CLOSE_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/data/hbase/meta/1588230740/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-11-11T03:18:24,516 DEBUG [RS_CLOSE_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-11T03:18:24,516 INFO [RS_CLOSE_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-11-11T03:18:24,516 DEBUG [RS_CLOSE_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-11-11T03:18:24,516 DEBUG [RS_CLOSE_META-regionserver/e4cdd3a0b883:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-11T03:18:24,593 INFO [regionserver/e4cdd3a0b883:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-11T03:18:24,593 INFO [regionserver/e4cdd3a0b883:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-11T03:18:24,666 INFO [RS:0;e4cdd3a0b883:41101 {}] regionserver.HRegionServer(1250): stopping server e4cdd3a0b883,41101,1731295033876; all regions closed. 2024-11-11T03:18:24,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43237 is added to blk_1073741834_1010 (size=2484) 2024-11-11T03:18:24,679 DEBUG [RS:0;e4cdd3a0b883:41101 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/oldWALs 2024-11-11T03:18:24,679 INFO [RS:0;e4cdd3a0b883:41101 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL e4cdd3a0b883%2C41101%2C1731295033876.meta:.meta(num 1731295037428) 2024-11-11T03:18:24,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43237 is added to blk_1073741833_1009 (size=1414) 2024-11-11T03:18:24,686 DEBUG [RS:0;e4cdd3a0b883:41101 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/oldWALs 2024-11-11T03:18:24,686 INFO [RS:0;e4cdd3a0b883:41101 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL e4cdd3a0b883%2C41101%2C1731295033876:(num 1731295036784) 2024-11-11T03:18:24,686 DEBUG [RS:0;e4cdd3a0b883:41101 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:18:24,687 INFO [RS:0;e4cdd3a0b883:41101 {}] regionserver.LeaseManager(133): Closed leases 2024-11-11T03:18:24,688 INFO [RS:0;e4cdd3a0b883:41101 {}] hbase.ChoreService(370): Chore service for: regionserver/e4cdd3a0b883:0 had [ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-11T03:18:24,688 INFO [regionserver/e4cdd3a0b883:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-11-11T03:18:24,689 INFO [RS:0;e4cdd3a0b883:41101 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:41101 2024-11-11T03:18:24,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/e4cdd3a0b883,41101,1731295033876 2024-11-11T03:18:24,694 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-11T03:18:24,696 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [e4cdd3a0b883,41101,1731295033876] 2024-11-11T03:18:24,696 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing e4cdd3a0b883,41101,1731295033876; numProcessing=1 2024-11-11T03:18:24,698 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/e4cdd3a0b883,41101,1731295033876 already deleted, retry=false 2024-11-11T03:18:24,698 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; e4cdd3a0b883,41101,1731295033876 expired; onlineServers=0 2024-11-11T03:18:24,698 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server 'e4cdd3a0b883,36699,1731295032496' ***** 2024-11-11T03:18:24,698 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-11T03:18:24,698 DEBUG [M:0;e4cdd3a0b883:36699 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f8babc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=e4cdd3a0b883/172.17.0.2:0 2024-11-11T03:18:24,699 INFO [M:0;e4cdd3a0b883:36699 {}] regionserver.HRegionServer(1224): stopping server e4cdd3a0b883,36699,1731295032496 2024-11-11T03:18:24,699 INFO [M:0;e4cdd3a0b883:36699 {}] regionserver.HRegionServer(1250): stopping server e4cdd3a0b883,36699,1731295032496; all regions closed. 2024-11-11T03:18:24,699 DEBUG [M:0;e4cdd3a0b883:36699 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-11T03:18:24,699 DEBUG [M:0;e4cdd3a0b883:36699 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-11T03:18:24,699 DEBUG [M:0;e4cdd3a0b883:36699 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-11T03:18:24,699 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-11T03:18:24,699 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster-HFileCleaner.small.0-1731295036213 {}] cleaner.HFileCleaner(306): Exit Thread[master/e4cdd3a0b883:0:becomeActiveMaster-HFileCleaner.small.0-1731295036213,5,FailOnTimeoutGroup] 2024-11-11T03:18:24,699 DEBUG [master/e4cdd3a0b883:0:becomeActiveMaster-HFileCleaner.large.0-1731295036186 {}] cleaner.HFileCleaner(306): Exit Thread[master/e4cdd3a0b883:0:becomeActiveMaster-HFileCleaner.large.0-1731295036186,5,FailOnTimeoutGroup] 2024-11-11T03:18:24,699 INFO [M:0;e4cdd3a0b883:36699 {}] hbase.ChoreService(370): Chore service for: master/e4cdd3a0b883:0 had [] on shutdown 2024-11-11T03:18:24,699 DEBUG [M:0;e4cdd3a0b883:36699 {}] master.HMaster(1733): Stopping service threads 2024-11-11T03:18:24,699 INFO [M:0;e4cdd3a0b883:36699 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-11T03:18:24,700 INFO [M:0;e4cdd3a0b883:36699 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-11T03:18:24,700 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-11T03:18:24,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-11T03:18:24,704 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-11T03:18:24,708 DEBUG [M:0;e4cdd3a0b883:36699 {}] zookeeper.ZKUtil(347): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-11T03:18:24,708 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-11T03:18:24,708 WARN [M:0;e4cdd3a0b883:36699 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-11T03:18:24,708 INFO [M:0;e4cdd3a0b883:36699 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-11-11T03:18:24,708 INFO [M:0;e4cdd3a0b883:36699 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-11T03:18:24,708 DEBUG [M:0;e4cdd3a0b883:36699 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-11T03:18:24,709 INFO [M:0;e4cdd3a0b883:36699 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T03:18:24,709 DEBUG [M:0;e4cdd3a0b883:36699 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T03:18:24,709 DEBUG [M:0;e4cdd3a0b883:36699 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-11T03:18:24,709 DEBUG [M:0;e4cdd3a0b883:36699 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T03:18:24,709 INFO [M:0;e4cdd3a0b883:36699 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=750.00 KB heapSize=899.16 KB 2024-11-11T03:18:24,731 DEBUG [M:0;e4cdd3a0b883:36699 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2f0dca9af373484682339f2175cc979e is 82, key is hbase:meta,,1/info:regioninfo/1731295037760/Put/seqid=0 2024-11-11T03:18:24,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43237 is added to blk_1073741840_1016 (size=5672) 2024-11-11T03:18:24,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:24,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:41101-0x101955526e60001, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:24,796 INFO [RS:0;e4cdd3a0b883:41101 {}] regionserver.HRegionServer(1307): Exiting; stopping=e4cdd3a0b883,41101,1731295033876; zookeeper connection closed. 2024-11-11T03:18:24,796 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@331dead1 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@331dead1 2024-11-11T03:18:24,797 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-11T03:18:25,140 INFO [M:0;e4cdd3a0b883:36699 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=1814 (bloomFilter=true), to=hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2f0dca9af373484682339f2175cc979e 2024-11-11T03:18:25,192 DEBUG [M:0;e4cdd3a0b883:36699 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a8b0eda8bc884fbdb31bc761bdc45983 is 2089, key is \x00\x00\x00\x00\x00\x00\x00\x9D/proc:d/1731295078805/Put/seqid=0 2024-11-11T03:18:25,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43237 is added to blk_1073741841_1017 (size=168558) 2024-11-11T03:18:25,598 INFO [M:0;e4cdd3a0b883:36699 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=747.45 KB at sequenceid=1814 (bloomFilter=true), to=hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a8b0eda8bc884fbdb31bc761bdc45983 2024-11-11T03:18:25,607 INFO [M:0;e4cdd3a0b883:36699 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a8b0eda8bc884fbdb31bc761bdc45983 2024-11-11T03:18:25,630 DEBUG [M:0;e4cdd3a0b883:36699 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7a752828fa404a03bea19e755f4845af is 69, key is e4cdd3a0b883,41101,1731295033876/rs:state/1731295036407/Put/seqid=0 2024-11-11T03:18:25,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43237 is added to blk_1073741842_1018 (size=5156) 2024-11-11T03:18:25,637 INFO [M:0;e4cdd3a0b883:36699 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=1814 (bloomFilter=true), to=hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7a752828fa404a03bea19e755f4845af 2024-11-11T03:18:25,671 DEBUG [M:0;e4cdd3a0b883:36699 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/667d9d03d07447859b292414a912f2d0 is 68, key is replication_peer_modification_on/state:d/1731295103828/Put/seqid=0 2024-11-11T03:18:25,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43237 is added to blk_1073741843_1019 (size=5154) 2024-11-11T03:18:25,677 INFO [M:0;e4cdd3a0b883:36699 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2 KB at sequenceid=1814 (bloomFilter=true), to=hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/667d9d03d07447859b292414a912f2d0 2024-11-11T03:18:25,686 DEBUG [M:0;e4cdd3a0b883:36699 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/2f0dca9af373484682339f2175cc979e as hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2f0dca9af373484682339f2175cc979e 2024-11-11T03:18:25,695 INFO [M:0;e4cdd3a0b883:36699 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/2f0dca9af373484682339f2175cc979e, entries=8, sequenceid=1814, filesize=5.5 K 2024-11-11T03:18:25,696 DEBUG [M:0;e4cdd3a0b883:36699 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/a8b0eda8bc884fbdb31bc761bdc45983 as hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a8b0eda8bc884fbdb31bc761bdc45983 2024-11-11T03:18:25,704 INFO [M:0;e4cdd3a0b883:36699 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for a8b0eda8bc884fbdb31bc761bdc45983 2024-11-11T03:18:25,704 INFO [M:0;e4cdd3a0b883:36699 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/a8b0eda8bc884fbdb31bc761bdc45983, entries=256, sequenceid=1814, filesize=164.6 K 2024-11-11T03:18:25,706 DEBUG [M:0;e4cdd3a0b883:36699 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/7a752828fa404a03bea19e755f4845af as hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7a752828fa404a03bea19e755f4845af 2024-11-11T03:18:25,716 INFO [M:0;e4cdd3a0b883:36699 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/7a752828fa404a03bea19e755f4845af, entries=1, sequenceid=1814, filesize=5.0 K 2024-11-11T03:18:25,717 DEBUG [M:0;e4cdd3a0b883:36699 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/667d9d03d07447859b292414a912f2d0 as hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/667d9d03d07447859b292414a912f2d0 2024-11-11T03:18:25,726 INFO [M:0;e4cdd3a0b883:36699 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:35635/user/jenkins/test-data/e53c0f54-35fe-81df-aa73-b50bf28eb6b3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/667d9d03d07447859b292414a912f2d0, entries=1, sequenceid=1814, filesize=5.0 K 2024-11-11T03:18:25,727 INFO [M:0;e4cdd3a0b883:36699 {}] regionserver.HRegion(3040): Finished flush of dataSize ~750.00 KB/768002, heapSize ~899.09 KB/920672, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1018ms, sequenceid=1814, compaction requested=false 2024-11-11T03:18:25,729 INFO [M:0;e4cdd3a0b883:36699 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-11T03:18:25,729 DEBUG [M:0;e4cdd3a0b883:36699 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-11T03:18:25,731 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43237 is added to blk_1073741830_1006 (size=887137) 2024-11-11T03:18:25,732 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-11-11T03:18:25,732 INFO [M:0;e4cdd3a0b883:36699 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-11-11T03:18:25,733 INFO [M:0;e4cdd3a0b883:36699 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:36699 2024-11-11T03:18:25,738 DEBUG [M:0;e4cdd3a0b883:36699 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/e4cdd3a0b883,36699,1731295032496 already deleted, retry=false 2024-11-11T03:18:25,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:25,841 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36699-0x101955526e60000, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:25,842 INFO [M:0;e4cdd3a0b883:36699 {}] regionserver.HRegionServer(1307): Exiting; stopping=e4cdd3a0b883,36699,1731295032496; zookeeper connection closed. 2024-11-11T03:18:25,944 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x101955526e60019, quorum=127.0.0.1:54139, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-11T03:18:25,950 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@19f1b88e{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-11T03:18:25,953 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@2117487{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T03:18:25,953 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T03:18:25,953 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@50244d8f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T03:18:25,953 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5b9056db{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/hadoop.log.dir/,STOPPED} 2024-11-11T03:18:25,956 WARN [BP-669106914-172.17.0.2-1731295029668 heartbeating to localhost/127.0.0.1:35635 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-11T03:18:25,957 WARN [BP-669106914-172.17.0.2-1731295029668 heartbeating to localhost/127.0.0.1:35635 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-669106914-172.17.0.2-1731295029668 (Datanode Uuid 90015427-e251-401a-9d78-c5ff268e48fd) service to localhost/127.0.0.1:35635 2024-11-11T03:18:25,958 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/cluster_ed1c06ec-b623-8234-3e18-7586117bca1f/dfs/data/data1/current/BP-669106914-172.17.0.2-1731295029668 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T03:18:25,959 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/cluster_ed1c06ec-b623-8234-3e18-7586117bca1f/dfs/data/data2/current/BP-669106914-172.17.0.2-1731295029668 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-11T03:18:25,959 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-11T03:18:25,959 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-11T03:18:25,960 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-11T03:18:25,968 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2490b214{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-11T03:18:25,968 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@1f7e2804{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-11T03:18:25,968 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-11T03:18:25,969 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@201ce359{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-11T03:18:25,969 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@842d13f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/3307ab6e-4d9b-ac87-effe-4743337da221/hadoop.log.dir/,STOPPED} 2024-11-11T03:18:25,981 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-11-11T03:18:26,003 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down