2024-12-09 03:19:48,357 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@302c971f 2024-12-09 03:19:48,387 main DEBUG Took 0.026124 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-12-09 03:19:48,387 main DEBUG PluginManager 'Core' found 129 plugins 2024-12-09 03:19:48,388 main DEBUG PluginManager 'Level' found 0 plugins 2024-12-09 03:19:48,390 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-12-09 03:19:48,392 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:19:48,427 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-12-09 03:19:48,443 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:19:48,446 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:19:48,447 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:19:48,447 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:19:48,448 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:19:48,448 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:19:48,450 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:19:48,450 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:19:48,451 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:19:48,451 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:19:48,453 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:19:48,453 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:19:48,454 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:19:48,455 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:19:48,455 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:19:48,456 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:19:48,457 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:19:48,457 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:19:48,458 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:19:48,458 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:19:48,459 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:19:48,460 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:19:48,461 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:19:48,461 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-12-09 03:19:48,463 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:19:48,463 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-12-09 03:19:48,473 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-12-09 03:19:48,477 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-12-09 03:19:48,479 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-12-09 03:19:48,480 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-12-09 03:19:48,484 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-12-09 03:19:48,485 main DEBUG PluginManager 'Converter' found 47 plugins 2024-12-09 03:19:48,504 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-12-09 03:19:48,509 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-12-09 03:19:48,512 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-12-09 03:19:48,512 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-12-09 03:19:48,513 main DEBUG createAppenders(={Console}) 2024-12-09 03:19:48,514 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@302c971f initialized 2024-12-09 03:19:48,514 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@302c971f 2024-12-09 03:19:48,515 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@302c971f OK. 2024-12-09 03:19:48,516 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-12-09 03:19:48,516 main DEBUG OutputStream closed 2024-12-09 03:19:48,517 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-12-09 03:19:48,517 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-12-09 03:19:48,518 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@505fc5a4 OK 2024-12-09 03:19:48,653 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-12-09 03:19:48,656 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-12-09 03:19:48,658 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-12-09 03:19:48,660 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-12-09 03:19:48,660 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-12-09 03:19:48,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-12-09 03:19:48,661 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-12-09 03:19:48,662 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-12-09 03:19:48,662 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-12-09 03:19:48,663 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-12-09 03:19:48,664 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-12-09 03:19:48,664 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-12-09 03:19:48,665 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-12-09 03:19:48,665 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-12-09 03:19:48,666 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-12-09 03:19:48,666 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-12-09 03:19:48,667 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-12-09 03:19:48,668 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-12-09 03:19:48,671 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09 03:19:48,672 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@39de3d36) with optional ClassLoader: null 2024-12-09 03:19:48,672 main DEBUG Shutdown hook enabled. Registering a new one. 2024-12-09 03:19:48,675 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@39de3d36] started OK. 2024-12-09T03:19:49,210 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de 2024-12-09 03:19:49,216 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-12-09 03:19:49,216 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-12-09T03:19:49,257 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestAsyncReplicationAdminApi timeout: 26 mins 2024-12-09T03:19:49,317 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-12-09T03:19:49,332 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/cluster_27f4fc50-796d-9115-8208-caf375f5faf1, deleteOnExit=true 2024-12-09T03:19:49,337 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-12-09T03:19:49,339 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/test.cache.data in system properties and HBase conf 2024-12-09T03:19:49,340 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/hadoop.tmp.dir in system properties and HBase conf 2024-12-09T03:19:49,340 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/hadoop.log.dir in system properties and HBase conf 2024-12-09T03:19:49,343 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/mapreduce.cluster.local.dir in system properties and HBase conf 2024-12-09T03:19:49,344 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-12-09T03:19:49,344 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-12-09T03:19:49,503 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-12-09T03:19:49,670 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-12-09T03:19:49,680 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-12-09T03:19:49,681 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-12-09T03:19:49,683 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-12-09T03:19:49,684 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T03:19:49,686 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-12-09T03:19:49,687 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-12-09T03:19:49,693 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-12-09T03:19:49,695 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T03:19:49,699 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-12-09T03:19:49,700 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/nfs.dump.dir in system properties and HBase conf 2024-12-09T03:19:49,700 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/java.io.tmpdir in system properties and HBase conf 2024-12-09T03:19:49,701 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/dfs.journalnode.edits.dir in system properties and HBase conf 2024-12-09T03:19:49,701 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-12-09T03:19:49,702 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-12-09T03:19:51,013 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-12-09T03:19:51,160 INFO [Time-limited test {}] log.Log(170): Logging initialized @3936ms to org.eclipse.jetty.util.log.Slf4jLog 2024-12-09T03:19:51,257 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T03:19:51,345 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T03:19:51,379 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T03:19:51,380 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T03:19:51,382 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-09T03:19:51,398 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T03:19:51,402 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@1d9ceadd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/hadoop.log.dir/,AVAILABLE} 2024-12-09T03:19:51,403 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@f2948e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T03:19:51,705 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2b5eba6b{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/java.io.tmpdir/jetty-localhost-41097-hadoop-hdfs-3_4_1-tests_jar-_-any-16384226978497775956/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T03:19:51,722 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@79c1d5da{HTTP/1.1, (http/1.1)}{localhost:41097} 2024-12-09T03:19:51,722 INFO [Time-limited test {}] server.Server(415): Started @4499ms 2024-12-09T03:19:52,284 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-12-09T03:19:52,293 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-12-09T03:19:52,295 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-12-09T03:19:52,295 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-12-09T03:19:52,295 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-12-09T03:19:52,296 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5148c4c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/hadoop.log.dir/,AVAILABLE} 2024-12-09T03:19:52,297 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@3045cad6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-12-09T03:19:52,422 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@a1044e0{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/java.io.tmpdir/jetty-localhost-40769-hadoop-hdfs-3_4_1-tests_jar-_-any-18325962719181145508/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T03:19:52,423 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@240f5d3{HTTP/1.1, (http/1.1)}{localhost:40769} 2024-12-09T03:19:52,424 INFO [Time-limited test {}] server.Server(415): Started @5201ms 2024-12-09T03:19:52,483 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-12-09T03:19:53,005 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/cluster_27f4fc50-796d-9115-8208-caf375f5faf1/dfs/data/data2/current/BP-733451773-172.17.0.2-1733714390664/current, will proceed with Du for space computation calculation, 2024-12-09T03:19:53,005 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/cluster_27f4fc50-796d-9115-8208-caf375f5faf1/dfs/data/data1/current/BP-733451773-172.17.0.2-1733714390664/current, will proceed with Du for space computation calculation, 2024-12-09T03:19:53,052 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-12-09T03:19:53,110 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7fd1df2c0e4a44ba with lease ID 0x64b31f431768c680: Processing first storage report for DS-2fa9c509-d93b-4e00-acad-fa2f242a0fce from datanode DatanodeRegistration(127.0.0.1:43681, datanodeUuid=0a9fb78d-b6f6-407b-a4c8-2f4f21e55997, infoPort=38561, infoSecurePort=0, ipcPort=35721, storageInfo=lv=-57;cid=testClusterID;nsid=2071148782;c=1733714390664) 2024-12-09T03:19:53,111 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7fd1df2c0e4a44ba with lease ID 0x64b31f431768c680: from storage DS-2fa9c509-d93b-4e00-acad-fa2f242a0fce node DatanodeRegistration(127.0.0.1:43681, datanodeUuid=0a9fb78d-b6f6-407b-a4c8-2f4f21e55997, infoPort=38561, infoSecurePort=0, ipcPort=35721, storageInfo=lv=-57;cid=testClusterID;nsid=2071148782;c=1733714390664), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-09T03:19:53,112 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x7fd1df2c0e4a44ba with lease ID 0x64b31f431768c680: Processing first storage report for DS-adc4c5a9-b816-4d28-9745-8d9e7059eb62 from datanode DatanodeRegistration(127.0.0.1:43681, datanodeUuid=0a9fb78d-b6f6-407b-a4c8-2f4f21e55997, infoPort=38561, infoSecurePort=0, ipcPort=35721, storageInfo=lv=-57;cid=testClusterID;nsid=2071148782;c=1733714390664) 2024-12-09T03:19:53,112 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x7fd1df2c0e4a44ba with lease ID 0x64b31f431768c680: from storage DS-adc4c5a9-b816-4d28-9745-8d9e7059eb62 node DatanodeRegistration(127.0.0.1:43681, datanodeUuid=0a9fb78d-b6f6-407b-a4c8-2f4f21e55997, infoPort=38561, infoSecurePort=0, ipcPort=35721, storageInfo=lv=-57;cid=testClusterID;nsid=2071148782;c=1733714390664), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-12-09T03:19:53,200 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de 2024-12-09T03:19:53,295 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/cluster_27f4fc50-796d-9115-8208-caf375f5faf1/zookeeper_0, clientPort=63605, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/cluster_27f4fc50-796d-9115-8208-caf375f5faf1/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/cluster_27f4fc50-796d-9115-8208-caf375f5faf1/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-12-09T03:19:53,307 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=63605 2024-12-09T03:19:53,323 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T03:19:53,327 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T03:19:53,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43681 is added to blk_1073741825_1001 (size=7) 2024-12-09T03:19:54,070 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324 with version=8 2024-12-09T03:19:54,070 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/hbase-staging 2024-12-09T03:19:54,206 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-12-09T03:19:54,493 INFO [Time-limited test {}] client.ConnectionUtils(129): master/66b6cc25935e:0 server-side Connection retries=6 2024-12-09T03:19:54,512 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T03:19:54,513 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T03:19:54,513 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T03:19:54,513 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T03:19:54,513 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T03:19:54,645 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T03:19:54,706 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-12-09T03:19:54,716 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-12-09T03:19:54,720 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T03:19:54,751 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 10925 (auto-detected) 2024-12-09T03:19:54,752 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-12-09T03:19:54,773 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:45333 2024-12-09T03:19:54,782 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T03:19:54,784 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T03:19:54,798 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:45333 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:19:54,835 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:453330x0, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:19:54,837 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45333-0x10074c9f1960000 connected 2024-12-09T03:19:54,877 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T03:19:54,880 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T03:19:54,883 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T03:19:54,887 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45333 2024-12-09T03:19:54,888 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45333 2024-12-09T03:19:54,891 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45333 2024-12-09T03:19:54,892 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45333 2024-12-09T03:19:54,892 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45333 2024-12-09T03:19:54,899 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324, hbase.cluster.distributed=false 2024-12-09T03:19:54,962 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/66b6cc25935e:0 server-side Connection retries=6 2024-12-09T03:19:54,963 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T03:19:54,963 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-12-09T03:19:54,963 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-12-09T03:19:54,963 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-12-09T03:19:54,963 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-12-09T03:19:54,965 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-12-09T03:19:54,967 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-12-09T03:19:54,968 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:45493 2024-12-09T03:19:54,970 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-12-09T03:19:54,975 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-12-09T03:19:54,977 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T03:19:54,980 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T03:19:54,983 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:45493 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:19:54,987 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:454930x0, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:19:54,987 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:454930x0, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T03:19:54,988 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45493-0x10074c9f1960001 connected 2024-12-09T03:19:54,989 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T03:19:54,990 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-12-09T03:19:54,992 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45493 2024-12-09T03:19:54,994 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45493 2024-12-09T03:19:54,995 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45493 2024-12-09T03:19:54,995 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45493 2024-12-09T03:19:54,996 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45493 2024-12-09T03:19:55,000 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/66b6cc25935e,45333,1733714394200 2024-12-09T03:19:55,008 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T03:19:55,008 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T03:19:55,012 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/66b6cc25935e,45333,1733714394200 2024-12-09T03:19:55,020 DEBUG [M:0;66b6cc25935e:45333 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;66b6cc25935e:45333 2024-12-09T03:19:55,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T03:19:55,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-12-09T03:19:55,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:19:55,034 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:19:55,035 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T03:19:55,036 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-12-09T03:19:55,037 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/66b6cc25935e,45333,1733714394200 from backup master directory 2024-12-09T03:19:55,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/66b6cc25935e,45333,1733714394200 2024-12-09T03:19:55,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T03:19:55,040 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-12-09T03:19:55,041 WARN [master/66b6cc25935e:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T03:19:55,041 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=66b6cc25935e,45333,1733714394200 2024-12-09T03:19:55,044 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-12-09T03:19:55,046 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-12-09T03:19:55,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43681 is added to blk_1073741826_1002 (size=42) 2024-12-09T03:19:55,518 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/hbase.id with ID: 3b7d4240-16c9-429e-aa98-985bf779f27b 2024-12-09T03:19:55,564 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-12-09T03:19:55,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:19:55,593 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:19:55,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43681 is added to blk_1073741827_1003 (size=196) 2024-12-09T03:19:56,031 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-12-09T03:19:56,034 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-12-09T03:19:56,060 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:402) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:973) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2470) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:590) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:587) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:19:56,066 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:19:56,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43681 is added to blk_1073741828_1004 (size=1189) 2024-12-09T03:19:56,516 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/data/master/store 2024-12-09T03:19:56,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43681 is added to blk_1073741829_1005 (size=34) 2024-12-09T03:19:56,536 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-12-09T03:19:56,537 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:19:56,538 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T03:19:56,538 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T03:19:56,538 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T03:19:56,538 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T03:19:56,538 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T03:19:56,538 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T03:19:56,538 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-09T03:19:56,540 WARN [master/66b6cc25935e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/data/master/store/.initializing 2024-12-09T03:19:56,541 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/WALs/66b6cc25935e,45333,1733714394200 2024-12-09T03:19:56,547 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T03:19:56,557 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=66b6cc25935e%2C45333%2C1733714394200, suffix=, logDir=hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/WALs/66b6cc25935e,45333,1733714394200, archiveDir=hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/oldWALs, maxLogs=10 2024-12-09T03:19:56,579 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(617): When create output stream for /user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/WALs/66b6cc25935e,45333,1733714394200/66b6cc25935e%2C45333%2C1733714394200.1733714396562, exclude list is [], retry=0 2024-12-09T03:19:56,594 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43681,DS-2fa9c509-d93b-4e00-acad-fa2f242a0fce,DISK] 2024-12-09T03:19:56,598 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-12-09T03:19:56,638 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/WALs/66b6cc25935e,45333,1733714394200/66b6cc25935e%2C45333%2C1733714394200.1733714396562 2024-12-09T03:19:56,638 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38561:38561)] 2024-12-09T03:19:56,639 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:19:56,639 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:19:56,643 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:19:56,643 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:19:56,684 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:19:56,711 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-12-09T03:19:56,715 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:19:56,719 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T03:19:56,719 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:19:56,723 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-12-09T03:19:56,723 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:19:56,725 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:19:56,725 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:19:56,728 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-12-09T03:19:56,729 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:19:56,730 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:19:56,730 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:19:56,733 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-12-09T03:19:56,733 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:19:56,734 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:19:56,739 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:19:56,740 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:19:56,749 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-12-09T03:19:56,754 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-12-09T03:19:56,759 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T03:19:56,760 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64319018, jitterRate=-0.04157194495201111}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-12-09T03:19:56,764 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-09T03:19:56,765 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-12-09T03:19:56,794 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a0b17f7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:19:56,827 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-12-09T03:19:56,839 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-12-09T03:19:56,839 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-12-09T03:19:56,841 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-12-09T03:19:56,843 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 1 msec 2024-12-09T03:19:56,848 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 4 msec 2024-12-09T03:19:56,848 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-12-09T03:19:56,873 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-12-09T03:19:56,885 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-12-09T03:19:56,887 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-12-09T03:19:56,889 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-12-09T03:19:56,891 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-12-09T03:19:56,893 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-12-09T03:19:56,895 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-12-09T03:19:56,898 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-12-09T03:19:56,900 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-12-09T03:19:56,901 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-12-09T03:19:56,903 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-12-09T03:19:56,915 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-12-09T03:19:56,916 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-12-09T03:19:56,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T03:19:56,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-12-09T03:19:56,921 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:19:56,922 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:19:56,922 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=66b6cc25935e,45333,1733714394200, sessionid=0x10074c9f1960000, setting cluster-up flag (Was=false) 2024-12-09T03:19:56,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:19:56,938 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:19:56,945 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-12-09T03:19:56,947 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=66b6cc25935e,45333,1733714394200 2024-12-09T03:19:56,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:19:56,952 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:19:56,958 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-12-09T03:19:56,959 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=66b6cc25935e,45333,1733714394200 2024-12-09T03:19:57,015 DEBUG [RS:0;66b6cc25935e:45493 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;66b6cc25935e:45493 2024-12-09T03:19:57,017 INFO [RS:0;66b6cc25935e:45493 {}] regionserver.HRegionServer(1008): ClusterId : 3b7d4240-16c9-429e-aa98-985bf779f27b 2024-12-09T03:19:57,020 DEBUG [RS:0;66b6cc25935e:45493 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-12-09T03:19:57,025 DEBUG [RS:0;66b6cc25935e:45493 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-12-09T03:19:57,025 DEBUG [RS:0;66b6cc25935e:45493 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-12-09T03:19:57,028 DEBUG [RS:0;66b6cc25935e:45493 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-12-09T03:19:57,029 DEBUG [RS:0;66b6cc25935e:45493 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d9cb772, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:19:57,031 DEBUG [RS:0;66b6cc25935e:45493 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@312dba52, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=66b6cc25935e/172.17.0.2:0 2024-12-09T03:19:57,036 INFO [RS:0;66b6cc25935e:45493 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-12-09T03:19:57,036 INFO [RS:0;66b6cc25935e:45493 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-12-09T03:19:57,036 DEBUG [RS:0;66b6cc25935e:45493 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-12-09T03:19:57,038 INFO [RS:0;66b6cc25935e:45493 {}] regionserver.HRegionServer(3073): reportForDuty to master=66b6cc25935e,45333,1733714394200 with isa=66b6cc25935e/172.17.0.2:45493, startcode=1733714394961 2024-12-09T03:19:57,043 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-12-09T03:19:57,048 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-12-09T03:19:57,050 DEBUG [RS:0;66b6cc25935e:45493 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T03:19:57,051 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-12-09T03:19:57,057 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 66b6cc25935e,45333,1733714394200 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-12-09T03:19:57,060 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/66b6cc25935e:0, corePoolSize=5, maxPoolSize=5 2024-12-09T03:19:57,060 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/66b6cc25935e:0, corePoolSize=5, maxPoolSize=5 2024-12-09T03:19:57,061 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/66b6cc25935e:0, corePoolSize=5, maxPoolSize=5 2024-12-09T03:19:57,061 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/66b6cc25935e:0, corePoolSize=5, maxPoolSize=5 2024-12-09T03:19:57,061 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/66b6cc25935e:0, corePoolSize=10, maxPoolSize=10 2024-12-09T03:19:57,061 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/66b6cc25935e:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:19:57,062 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/66b6cc25935e:0, corePoolSize=2, maxPoolSize=2 2024-12-09T03:19:57,062 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/66b6cc25935e:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:19:57,065 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1733714427065 2024-12-09T03:19:57,067 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-12-09T03:19:57,067 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-12-09T03:19:57,068 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-12-09T03:19:57,068 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-12-09T03:19:57,072 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-12-09T03:19:57,073 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-12-09T03:19:57,073 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-12-09T03:19:57,074 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:19:57,074 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-12-09T03:19:57,074 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T03:19:57,077 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T03:19:57,079 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-12-09T03:19:57,081 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-12-09T03:19:57,081 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-12-09T03:19:57,084 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-12-09T03:19:57,085 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-12-09T03:19:57,086 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/66b6cc25935e:0:becomeActiveMaster-HFileCleaner.large.0-1733714397086,5,FailOnTimeoutGroup] 2024-12-09T03:19:57,086 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/66b6cc25935e:0:becomeActiveMaster-HFileCleaner.small.0-1733714397086,5,FailOnTimeoutGroup] 2024-12-09T03:19:57,086 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-12-09T03:19:57,087 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-12-09T03:19:57,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43681 is added to blk_1073741831_1007 (size=1039) 2024-12-09T03:19:57,089 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-12-09T03:19:57,089 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-12-09T03:19:57,090 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44381, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T03:19:57,095 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45333 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 66b6cc25935e,45493,1733714394961 2024-12-09T03:19:57,098 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45333 {}] master.ServerManager(486): Registering regionserver=66b6cc25935e,45493,1733714394961 2024-12-09T03:19:57,114 DEBUG [RS:0;66b6cc25935e:45493 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324 2024-12-09T03:19:57,114 DEBUG [RS:0;66b6cc25935e:45493 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:43899 2024-12-09T03:19:57,114 DEBUG [RS:0;66b6cc25935e:45493 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-12-09T03:19:57,119 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T03:19:57,119 DEBUG [RS:0;66b6cc25935e:45493 {}] zookeeper.ZKUtil(111): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/66b6cc25935e,45493,1733714394961 2024-12-09T03:19:57,120 WARN [RS:0;66b6cc25935e:45493 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-12-09T03:19:57,120 INFO [RS:0;66b6cc25935e:45493 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:19:57,120 DEBUG [RS:0;66b6cc25935e:45493 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/WALs/66b6cc25935e,45493,1733714394961 2024-12-09T03:19:57,122 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [66b6cc25935e,45493,1733714394961] 2024-12-09T03:19:57,136 DEBUG [RS:0;66b6cc25935e:45493 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-12-09T03:19:57,151 INFO [RS:0;66b6cc25935e:45493 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-12-09T03:19:57,167 INFO [RS:0;66b6cc25935e:45493 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-12-09T03:19:57,171 INFO [RS:0;66b6cc25935e:45493 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-12-09T03:19:57,171 INFO [RS:0;66b6cc25935e:45493 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T03:19:57,172 INFO [RS:0;66b6cc25935e:45493 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-12-09T03:19:57,180 INFO [RS:0;66b6cc25935e:45493 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-12-09T03:19:57,181 DEBUG [RS:0;66b6cc25935e:45493 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/66b6cc25935e:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:19:57,181 DEBUG [RS:0;66b6cc25935e:45493 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/66b6cc25935e:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:19:57,181 DEBUG [RS:0;66b6cc25935e:45493 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/66b6cc25935e:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:19:57,181 DEBUG [RS:0;66b6cc25935e:45493 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/66b6cc25935e:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:19:57,182 DEBUG [RS:0;66b6cc25935e:45493 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/66b6cc25935e:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:19:57,182 DEBUG [RS:0;66b6cc25935e:45493 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/66b6cc25935e:0, corePoolSize=2, maxPoolSize=2 2024-12-09T03:19:57,182 DEBUG [RS:0;66b6cc25935e:45493 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/66b6cc25935e:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:19:57,182 DEBUG [RS:0;66b6cc25935e:45493 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/66b6cc25935e:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:19:57,182 DEBUG [RS:0;66b6cc25935e:45493 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/66b6cc25935e:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:19:57,183 DEBUG [RS:0;66b6cc25935e:45493 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/66b6cc25935e:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:19:57,183 DEBUG [RS:0;66b6cc25935e:45493 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/66b6cc25935e:0, corePoolSize=1, maxPoolSize=1 2024-12-09T03:19:57,183 DEBUG [RS:0;66b6cc25935e:45493 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/66b6cc25935e:0, corePoolSize=3, maxPoolSize=3 2024-12-09T03:19:57,183 DEBUG [RS:0;66b6cc25935e:45493 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/66b6cc25935e:0, corePoolSize=3, maxPoolSize=3 2024-12-09T03:19:57,184 INFO [RS:0;66b6cc25935e:45493 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T03:19:57,184 INFO [RS:0;66b6cc25935e:45493 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-12-09T03:19:57,185 INFO [RS:0;66b6cc25935e:45493 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-12-09T03:19:57,185 INFO [RS:0;66b6cc25935e:45493 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-12-09T03:19:57,185 INFO [RS:0;66b6cc25935e:45493 {}] hbase.ChoreService(168): Chore ScheduledChore name=66b6cc25935e,45493,1733714394961-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T03:19:57,215 INFO [RS:0;66b6cc25935e:45493 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-12-09T03:19:57,217 INFO [RS:0;66b6cc25935e:45493 {}] hbase.ChoreService(168): Chore ScheduledChore name=66b6cc25935e,45493,1733714394961-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T03:19:57,246 INFO [RS:0;66b6cc25935e:45493 {}] regionserver.Replication(204): 66b6cc25935e,45493,1733714394961 started 2024-12-09T03:19:57,246 INFO [RS:0;66b6cc25935e:45493 {}] regionserver.HRegionServer(1767): Serving as 66b6cc25935e,45493,1733714394961, RpcServer on 66b6cc25935e/172.17.0.2:45493, sessionid=0x10074c9f1960001 2024-12-09T03:19:57,247 DEBUG [RS:0;66b6cc25935e:45493 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-12-09T03:19:57,247 DEBUG [RS:0;66b6cc25935e:45493 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 66b6cc25935e,45493,1733714394961 2024-12-09T03:19:57,247 DEBUG [RS:0;66b6cc25935e:45493 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '66b6cc25935e,45493,1733714394961' 2024-12-09T03:19:57,247 DEBUG [RS:0;66b6cc25935e:45493 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-12-09T03:19:57,248 DEBUG [RS:0;66b6cc25935e:45493 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-12-09T03:19:57,249 DEBUG [RS:0;66b6cc25935e:45493 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-12-09T03:19:57,249 DEBUG [RS:0;66b6cc25935e:45493 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-12-09T03:19:57,249 DEBUG [RS:0;66b6cc25935e:45493 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 66b6cc25935e,45493,1733714394961 2024-12-09T03:19:57,250 DEBUG [RS:0;66b6cc25935e:45493 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '66b6cc25935e,45493,1733714394961' 2024-12-09T03:19:57,250 DEBUG [RS:0;66b6cc25935e:45493 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-12-09T03:19:57,250 DEBUG [RS:0;66b6cc25935e:45493 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-12-09T03:19:57,251 DEBUG [RS:0;66b6cc25935e:45493 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-12-09T03:19:57,251 INFO [RS:0;66b6cc25935e:45493 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-12-09T03:19:57,251 INFO [RS:0;66b6cc25935e:45493 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-12-09T03:19:57,359 INFO [RS:0;66b6cc25935e:45493 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-12-09T03:19:57,364 INFO [RS:0;66b6cc25935e:45493 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=66b6cc25935e%2C45493%2C1733714394961, suffix=, logDir=hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/WALs/66b6cc25935e,45493,1733714394961, archiveDir=hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/oldWALs, maxLogs=32 2024-12-09T03:19:57,387 DEBUG [RS:0;66b6cc25935e:45493 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(617): When create output stream for /user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/WALs/66b6cc25935e,45493,1733714394961/66b6cc25935e%2C45493%2C1733714394961.1733714397367, exclude list is [], retry=0 2024-12-09T03:19:57,393 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43681,DS-2fa9c509-d93b-4e00-acad-fa2f242a0fce,DISK] 2024-12-09T03:19:57,397 INFO [RS:0;66b6cc25935e:45493 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/WALs/66b6cc25935e,45493,1733714394961/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:19:57,398 DEBUG [RS:0;66b6cc25935e:45493 {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38561:38561)] 2024-12-09T03:19:57,491 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-12-09T03:19:57,491 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324 2024-12-09T03:19:57,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43681 is added to blk_1073741833_1009 (size=32) 2024-12-09T03:19:57,903 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:19:57,906 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T03:19:57,909 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T03:19:57,910 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:19:57,910 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T03:19:57,911 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T03:19:57,913 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T03:19:57,914 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:19:57,915 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T03:19:57,915 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T03:19:57,918 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T03:19:57,918 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:19:57,919 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T03:19:57,921 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/data/hbase/meta/1588230740 2024-12-09T03:19:57,922 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/data/hbase/meta/1588230740 2024-12-09T03:19:57,926 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T03:19:57,929 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-09T03:19:57,934 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T03:19:57,935 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62649261, jitterRate=-0.06645326316356659}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T03:19:57,939 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-09T03:19:57,939 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-09T03:19:57,939 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-09T03:19:57,939 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-09T03:19:57,939 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T03:19:57,939 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T03:19:57,941 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-09T03:19:57,941 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-09T03:19:57,944 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-12-09T03:19:57,944 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-12-09T03:19:57,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-12-09T03:19:57,962 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-12-09T03:19:57,966 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-12-09T03:19:58,118 DEBUG [66b6cc25935e:45333 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-12-09T03:19:58,123 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=66b6cc25935e,45493,1733714394961 2024-12-09T03:19:58,129 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 66b6cc25935e,45493,1733714394961, state=OPENING 2024-12-09T03:19:58,135 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-12-09T03:19:58,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:19:58,138 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:19:58,139 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T03:19:58,139 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T03:19:58,140 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=66b6cc25935e,45493,1733714394961}] 2024-12-09T03:19:58,316 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:19:58,318 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-12-09T03:19:58,321 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58560, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-12-09T03:19:58,332 INFO [RS_OPEN_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-12-09T03:19:58,333 INFO [RS_OPEN_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-12-09T03:19:58,333 INFO [RS_OPEN_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-12-09T03:19:58,336 INFO [RS_OPEN_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=66b6cc25935e%2C45493%2C1733714394961.meta, suffix=.meta, logDir=hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/WALs/66b6cc25935e,45493,1733714394961, archiveDir=hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/oldWALs, maxLogs=32 2024-12-09T03:19:58,353 DEBUG [RS_OPEN_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(617): When create output stream for /user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/WALs/66b6cc25935e,45493,1733714394961/66b6cc25935e%2C45493%2C1733714394961.meta.1733714398338.meta, exclude list is [], retry=0 2024-12-09T03:19:58,357 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:43681,DS-2fa9c509-d93b-4e00-acad-fa2f242a0fce,DISK] 2024-12-09T03:19:58,361 INFO [RS_OPEN_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/WALs/66b6cc25935e,45493,1733714394961/66b6cc25935e%2C45493%2C1733714394961.meta.1733714398338.meta 2024-12-09T03:19:58,361 DEBUG [RS_OPEN_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:38561:38561)] 2024-12-09T03:19:58,361 DEBUG [RS_OPEN_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:19:58,363 DEBUG [RS_OPEN_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-12-09T03:19:58,422 DEBUG [RS_OPEN_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-12-09T03:19:58,427 INFO [RS_OPEN_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-12-09T03:19:58,431 DEBUG [RS_OPEN_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-12-09T03:19:58,431 DEBUG [RS_OPEN_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:19:58,431 DEBUG [RS_OPEN_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-12-09T03:19:58,431 DEBUG [RS_OPEN_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-12-09T03:19:58,435 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-12-09T03:19:58,437 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-12-09T03:19:58,437 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:19:58,438 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T03:19:58,438 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-12-09T03:19:58,440 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-12-09T03:19:58,440 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:19:58,441 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T03:19:58,441 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-12-09T03:19:58,443 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-12-09T03:19:58,443 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:19:58,444 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-12-09T03:19:58,445 DEBUG [RS_OPEN_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/data/hbase/meta/1588230740 2024-12-09T03:19:58,448 DEBUG [RS_OPEN_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/data/hbase/meta/1588230740 2024-12-09T03:19:58,451 DEBUG [RS_OPEN_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-12-09T03:19:58,454 DEBUG [RS_OPEN_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-12-09T03:19:58,455 INFO [RS_OPEN_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59920128, jitterRate=-0.10712051391601562}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-12-09T03:19:58,457 DEBUG [RS_OPEN_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-12-09T03:19:58,465 INFO [RS_OPEN_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1733714398310 2024-12-09T03:19:58,476 DEBUG [RS_OPEN_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-12-09T03:19:58,477 INFO [RS_OPEN_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-12-09T03:19:58,478 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=66b6cc25935e,45493,1733714394961 2024-12-09T03:19:58,480 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 66b6cc25935e,45493,1733714394961, state=OPEN 2024-12-09T03:19:58,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T03:19:58,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-12-09T03:19:58,486 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T03:19:58,486 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-12-09T03:19:58,491 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-12-09T03:19:58,491 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=66b6cc25935e,45493,1733714394961 in 346 msec 2024-12-09T03:19:58,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-12-09T03:19:58,497 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 541 msec 2024-12-09T03:19:58,502 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.5080 sec 2024-12-09T03:19:58,502 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1733714398502, completionTime=-1 2024-12-09T03:19:58,503 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-12-09T03:19:58,503 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-12-09T03:19:58,549 DEBUG [hconnection-0x391cbaad-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T03:19:58,552 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58574, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T03:19:58,566 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-12-09T03:19:58,566 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1733714458566 2024-12-09T03:19:58,566 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1733714518566 2024-12-09T03:19:58,566 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 63 msec 2024-12-09T03:19:58,591 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=66b6cc25935e,45333,1733714394200-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-12-09T03:19:58,591 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=66b6cc25935e,45333,1733714394200-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T03:19:58,592 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=66b6cc25935e,45333,1733714394200-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T03:19:58,594 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-66b6cc25935e:45333, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T03:19:58,594 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-12-09T03:19:58,604 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-12-09T03:19:58,607 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-12-09T03:19:58,613 DEBUG [master/66b6cc25935e:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-12-09T03:19:58,617 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-12-09T03:19:58,622 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-12-09T03:19:58,623 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:19:58,626 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-12-09T03:19:58,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43681 is added to blk_1073741835_1011 (size=358) 2024-12-09T03:19:59,143 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => 0f8f456dea7eb3922ff1bb2b905c7aee, NAME => 'hbase:namespace,,1733714398605.0f8f456dea7eb3922ff1bb2b905c7aee.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324 2024-12-09T03:19:59,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43681 is added to blk_1073741836_1012 (size=42) 2024-12-09T03:19:59,209 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733714398605.0f8f456dea7eb3922ff1bb2b905c7aee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:19:59,209 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing 0f8f456dea7eb3922ff1bb2b905c7aee, disabling compactions & flushes 2024-12-09T03:19:59,209 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733714398605.0f8f456dea7eb3922ff1bb2b905c7aee. 2024-12-09T03:19:59,209 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733714398605.0f8f456dea7eb3922ff1bb2b905c7aee. 2024-12-09T03:19:59,209 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733714398605.0f8f456dea7eb3922ff1bb2b905c7aee. after waiting 0 ms 2024-12-09T03:19:59,209 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733714398605.0f8f456dea7eb3922ff1bb2b905c7aee. 2024-12-09T03:19:59,209 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1733714398605.0f8f456dea7eb3922ff1bb2b905c7aee. 2024-12-09T03:19:59,209 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for 0f8f456dea7eb3922ff1bb2b905c7aee: 2024-12-09T03:19:59,216 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-12-09T03:19:59,226 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1733714398605.0f8f456dea7eb3922ff1bb2b905c7aee.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1733714399217"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1733714399217"}]},"ts":"1733714399217"} 2024-12-09T03:19:59,274 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-12-09T03:19:59,278 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-12-09T03:19:59,282 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733714399278"}]},"ts":"1733714399278"} 2024-12-09T03:19:59,292 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-12-09T03:19:59,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=0f8f456dea7eb3922ff1bb2b905c7aee, ASSIGN}] 2024-12-09T03:19:59,309 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=0f8f456dea7eb3922ff1bb2b905c7aee, ASSIGN 2024-12-09T03:19:59,313 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=0f8f456dea7eb3922ff1bb2b905c7aee, ASSIGN; state=OFFLINE, location=66b6cc25935e,45493,1733714394961; forceNewPlan=false, retain=false 2024-12-09T03:19:59,464 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=0f8f456dea7eb3922ff1bb2b905c7aee, regionState=OPENING, regionLocation=66b6cc25935e,45493,1733714394961 2024-12-09T03:19:59,473 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure 0f8f456dea7eb3922ff1bb2b905c7aee, server=66b6cc25935e,45493,1733714394961}] 2024-12-09T03:19:59,632 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:19:59,642 INFO [RS_OPEN_PRIORITY_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1733714398605.0f8f456dea7eb3922ff1bb2b905c7aee. 2024-12-09T03:19:59,643 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => 0f8f456dea7eb3922ff1bb2b905c7aee, NAME => 'hbase:namespace,,1733714398605.0f8f456dea7eb3922ff1bb2b905c7aee.', STARTKEY => '', ENDKEY => ''} 2024-12-09T03:19:59,644 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace 0f8f456dea7eb3922ff1bb2b905c7aee 2024-12-09T03:19:59,644 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1733714398605.0f8f456dea7eb3922ff1bb2b905c7aee.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-12-09T03:19:59,644 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for 0f8f456dea7eb3922ff1bb2b905c7aee 2024-12-09T03:19:59,644 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for 0f8f456dea7eb3922ff1bb2b905c7aee 2024-12-09T03:19:59,648 INFO [StoreOpener-0f8f456dea7eb3922ff1bb2b905c7aee-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 0f8f456dea7eb3922ff1bb2b905c7aee 2024-12-09T03:19:59,651 INFO [StoreOpener-0f8f456dea7eb3922ff1bb2b905c7aee-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0f8f456dea7eb3922ff1bb2b905c7aee columnFamilyName info 2024-12-09T03:19:59,651 DEBUG [StoreOpener-0f8f456dea7eb3922ff1bb2b905c7aee-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-12-09T03:19:59,652 INFO [StoreOpener-0f8f456dea7eb3922ff1bb2b905c7aee-1 {}] regionserver.HStore(327): Store=0f8f456dea7eb3922ff1bb2b905c7aee/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-12-09T03:19:59,654 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/data/hbase/namespace/0f8f456dea7eb3922ff1bb2b905c7aee 2024-12-09T03:19:59,655 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/data/hbase/namespace/0f8f456dea7eb3922ff1bb2b905c7aee 2024-12-09T03:19:59,661 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for 0f8f456dea7eb3922ff1bb2b905c7aee 2024-12-09T03:19:59,670 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/data/hbase/namespace/0f8f456dea7eb3922ff1bb2b905c7aee/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-12-09T03:19:59,672 INFO [RS_OPEN_PRIORITY_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened 0f8f456dea7eb3922ff1bb2b905c7aee; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62582964, jitterRate=-0.06744116544723511}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-12-09T03:19:59,674 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for 0f8f456dea7eb3922ff1bb2b905c7aee: 2024-12-09T03:19:59,677 INFO [RS_OPEN_PRIORITY_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1733714398605.0f8f456dea7eb3922ff1bb2b905c7aee., pid=6, masterSystemTime=1733714399631 2024-12-09T03:19:59,681 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1733714398605.0f8f456dea7eb3922ff1bb2b905c7aee. 2024-12-09T03:19:59,682 INFO [RS_OPEN_PRIORITY_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1733714398605.0f8f456dea7eb3922ff1bb2b905c7aee. 2024-12-09T03:19:59,683 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=0f8f456dea7eb3922ff1bb2b905c7aee, regionState=OPEN, openSeqNum=2, regionLocation=66b6cc25935e,45493,1733714394961 2024-12-09T03:19:59,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-12-09T03:19:59,695 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure 0f8f456dea7eb3922ff1bb2b905c7aee, server=66b6cc25935e,45493,1733714394961 in 215 msec 2024-12-09T03:19:59,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-12-09T03:19:59,703 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=0f8f456dea7eb3922ff1bb2b905c7aee, ASSIGN in 391 msec 2024-12-09T03:19:59,705 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-12-09T03:19:59,705 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1733714399705"}]},"ts":"1733714399705"} 2024-12-09T03:19:59,709 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-12-09T03:19:59,715 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-12-09T03:19:59,720 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.1090 sec 2024-12-09T03:19:59,723 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-12-09T03:19:59,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-12-09T03:19:59,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:19:59,725 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:19:59,767 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-12-09T03:19:59,789 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-09T03:19:59,797 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 35 msec 2024-12-09T03:19:59,808 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-12-09T03:19:59,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-12-09T03:19:59,854 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 43 msec 2024-12-09T03:19:59,871 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-12-09T03:19:59,876 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-12-09T03:19:59,877 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 4.835sec 2024-12-09T03:19:59,878 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-12-09T03:19:59,882 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-12-09T03:19:59,883 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-12-09T03:19:59,884 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-12-09T03:19:59,884 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-12-09T03:19:59,886 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=66b6cc25935e,45333,1733714394200-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-12-09T03:19:59,887 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=66b6cc25935e,45333,1733714394200-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-12-09T03:19:59,915 DEBUG [master/66b6cc25935e:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-12-09T03:19:59,917 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-12-09T03:19:59,917 INFO [master/66b6cc25935e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=66b6cc25935e,45333,1733714394200-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-12-09T03:19:59,953 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x005c3638 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@54db0f21 2024-12-09T03:19:59,964 WARN [Time-limited test {}] client.ZKConnectionRegistry(90): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-12-09T03:20:00,010 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74f9e8ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:00,017 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-12-09T03:20:00,018 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-12-09T03:20:00,052 DEBUG [hconnection-0x36e2e049-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-12-09T03:20:00,081 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58576, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-12-09T03:20:00,097 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=66b6cc25935e,45333,1733714394200 2024-12-09T03:20:00,101 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0e81966d to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1cd7f603 2024-12-09T03:20:00,131 DEBUG [ReadOnlyZKClient-127.0.0.1:63605@0x0e81966d {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@401392e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:00,175 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[0] Thread=222, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=495, ProcessCount=11, AvailableMemoryMB=9675 2024-12-09T03:20:00,211 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching master stub from registry 2024-12-09T03:20:00,223 DEBUG [ReadOnlyZKClient-127.0.0.1:63605@0x0e81966d {}] client.AsyncConnectionImpl(310): The fetched master address is 66b6cc25935e,45333,1733714394200 2024-12-09T03:20:00,226 DEBUG [ReadOnlyZKClient-127.0.0.1:63605@0x0e81966d {}] client.ConnectionUtils(581): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@898b899 2024-12-09T03:20:00,227 DEBUG [ReadOnlyZKClient-127.0.0.1:63605@0x0e81966d {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-12-09T03:20:00,231 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39754, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-12-09T03:20:00,233 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:00,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:00,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-09T03:20:00,275 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:00,277 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:00,283 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:00,283 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960004 connected 2024-12-09T03:20:00,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-09T03:20:00,392 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960004, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:00,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:20:00,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:20:00,417 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:00,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-09T03:20:00,585 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:00,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=10 2024-12-09T03:20:00,588 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T03:20:00,619 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=10}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:00,652 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:00,686 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-12-09T03:20:00,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=10 2024-12-09T03:20:00,722 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:00,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-12-09T03:20:00,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 305 msec 2024-12-09T03:20:00,727 INFO [PEWorker-3 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T03:20:00,731 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 492 msec 2024-12-09T03:20:00,755 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7a728a01 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@39650780 2024-12-09T03:20:00,761 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:00,762 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:00,763 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@2fe287af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:00,765 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:00,799 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:00,799 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:00,800 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960006 connected 2024-12-09T03:20:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-12-09T03:20:00,879 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:00,886 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T03:20:00,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=11, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:00,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=11 2024-12-09T03:20:00,922 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:00,925 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:00,930 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:00,931 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960007 connected 2024-12-09T03:20:01,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=11 2024-12-09T03:20:01,034 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960007, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:01,041 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:01,196 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:01,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=12 2024-12-09T03:20:01,197 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:01,199 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:01,200 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:01,200 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:01,200 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960006, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:01,200 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960006, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:01,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=11 2024-12-09T03:20:01,303 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960006, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:01,304 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=12}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7a728a01 to 127.0.0.1:63605 2024-12-09T03:20:01,304 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=12}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:01,329 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:01,330 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:01,336 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-12-09T03:20:01,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=12 2024-12-09T03:20:01,342 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:01,347 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=12, resume processing ppid=11 2024-12-09T03:20:01,347 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, ppid=11, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 301 msec 2024-12-09T03:20:01,347 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T03:20:01,351 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 460 msec 2024-12-09T03:20:01,390 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x00574d43 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@78d7f1f4 2024-12-09T03:20:01,403 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:01,403 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:01,404 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@5ee93e5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:01,408 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:01,419 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:01,422 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:01,422 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960009 connected 2024-12-09T03:20:01,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=11 2024-12-09T03:20:01,527 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:01,533 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:01,534 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:01,536 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:01,540 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-09T03:20:01,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=13, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:01,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=13 2024-12-09T03:20:01,569 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:01,571 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:01,574 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:01,575 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196000a connected 2024-12-09T03:20:01,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=13 2024-12-09T03:20:01,677 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196000a, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:01,687 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:01,841 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:01,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=14 2024-12-09T03:20:01,842 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:01,846 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:01,846 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:01,846 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:01,847 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960009, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:01,847 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960009, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:01,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=13 2024-12-09T03:20:01,949 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960009, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:01,949 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=14}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x00574d43 to 127.0.0.1:63605 2024-12-09T03:20:01,950 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=14}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:01,970 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:01,971 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:01,971 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-12-09T03:20:01,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=14 2024-12-09T03:20:01,978 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:01,983 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=14, resume processing ppid=13 2024-12-09T03:20:01,983 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, ppid=13, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 291 msec 2024-12-09T03:20:01,983 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-09T03:20:01,986 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 443 msec 2024-12-09T03:20:02,017 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x583e0dd9 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7f7ba14b 2024-12-09T03:20:02,029 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:02,030 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:02,030 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@48edf0b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:02,033 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:02,038 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:02,038 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:02,039 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f196000c connected 2024-12-09T03:20:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=13 2024-12-09T03:20:02,168 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:02,169 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:02,173 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:02,175 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:02,176 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:02,177 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-09T03:20:02,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=15, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:02,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-12-09T03:20:02,203 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:02,204 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:02,207 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:02,208 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196000d connected 2024-12-09T03:20:02,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-12-09T03:20:02,310 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196000d, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:02,316 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:02,469 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:02,471 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=16 2024-12-09T03:20:02,471 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:02,473 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:02,473 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:02,473 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:02,473 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f196000c, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:02,474 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f196000c, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:02,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-12-09T03:20:02,576 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f196000c, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:02,576 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=16}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x583e0dd9 to 127.0.0.1:63605 2024-12-09T03:20:02,576 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=16}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:02,608 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:02,608 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:02,610 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-12-09T03:20:02,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=16 2024-12-09T03:20:02,617 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:02,625 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=16, resume processing ppid=15 2024-12-09T03:20:02,625 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, ppid=15, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 301 msec 2024-12-09T03:20:02,625 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-09T03:20:02,631 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 448 msec 2024-12-09T03:20:02,688 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x477f7f9c to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@f5cdeda 2024-12-09T03:20:02,695 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:02,697 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:02,697 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@5b14ad4b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:02,699 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:02,702 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:02,703 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:02,703 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f196000f connected 2024-12-09T03:20:02,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-12-09T03:20:02,808 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:02,809 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:02,810 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:02,812 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:02,813 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T03:20:02,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=17, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:02,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=17 2024-12-09T03:20:02,841 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:02,842 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:02,845 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:02,852 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960010 connected 2024-12-09T03:20:02,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=17 2024-12-09T03:20:02,955 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960010, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:02,962 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:03,115 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:03,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=18 2024-12-09T03:20:03,117 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:03,119 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:03,119 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:03,119 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:03,119 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f196000f, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:03,120 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f196000f, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:03,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=17 2024-12-09T03:20:03,142 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-12-09T03:20:03,144 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-12-09T03:20:03,223 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f196000f, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:03,223 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=18}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x477f7f9c to 127.0.0.1:63605 2024-12-09T03:20:03,224 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=18}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:03,247 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:03,248 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:03,248 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-12-09T03:20:03,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=18 2024-12-09T03:20:03,253 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:03,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=18, resume processing ppid=17 2024-12-09T03:20:03,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, ppid=17, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 291 msec 2024-12-09T03:20:03,257 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T03:20:03,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 444 msec 2024-12-09T03:20:03,286 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x366131a9 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1527dde3 2024-12-09T03:20:03,298 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:03,298 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:03,298 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@6761ff00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:03,300 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:03,303 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:03,304 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:03,312 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960012 connected 2024-12-09T03:20:03,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=17 2024-12-09T03:20:03,448 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:03,448 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:03,450 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:03,451 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-09T03:20:03,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=19, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:03,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=19 2024-12-09T03:20:03,473 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:03,474 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:03,478 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:03,488 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960013 connected 2024-12-09T03:20:03,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=19 2024-12-09T03:20:03,590 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960013, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:03,596 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:03,749 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:03,750 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=20 2024-12-09T03:20:03,751 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:03,753 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:03,753 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:03,753 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:03,753 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960012, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:03,753 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960012, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:03,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=19 2024-12-09T03:20:03,856 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960012, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:03,856 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=20}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x366131a9 to 127.0.0.1:63605 2024-12-09T03:20:03,856 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=20}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:03,876 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:03,877 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:03,877 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-12-09T03:20:03,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=20 2024-12-09T03:20:03,883 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:03,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=20, resume processing ppid=19 2024-12-09T03:20:03,888 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=20, ppid=19, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 287 msec 2024-12-09T03:20:03,888 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-09T03:20:03,891 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=19, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 437 msec 2024-12-09T03:20:03,923 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0a39a8be to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6668343a 2024-12-09T03:20:03,929 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:03,930 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:03,930 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@4101ca68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:03,932 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:03,935 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:03,935 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:03,936 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960015 connected 2024-12-09T03:20:03,944 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T03:20:04,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=19 2024-12-09T03:20:04,077 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:04,078 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:04,079 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T03:20:04,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=21, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:04,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-12-09T03:20:04,102 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:04,103 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:04,106 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:04,107 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960016 connected 2024-12-09T03:20:04,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-12-09T03:20:04,209 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960016, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:04,216 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:04,369 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:04,371 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=22 2024-12-09T03:20:04,371 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:04,373 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:04,373 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:04,373 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:04,373 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960015, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:04,374 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960015, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:04,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-12-09T03:20:04,476 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960015, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:04,477 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=22}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0a39a8be to 127.0.0.1:63605 2024-12-09T03:20:04,477 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=22}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:04,500 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:04,500 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:04,500 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-12-09T03:20:04,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=22 2024-12-09T03:20:04,506 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:04,509 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=22, resume processing ppid=21 2024-12-09T03:20:04,510 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=22, ppid=21, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 290 msec 2024-12-09T03:20:04,510 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T03:20:04,512 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=21, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 431 msec 2024-12-09T03:20:04,536 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3699cd67 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3d8b90d7 2024-12-09T03:20:04,541 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:04,541 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:04,541 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@32fd86ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:04,543 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:04,559 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:04,559 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:04,560 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960018 connected 2024-12-09T03:20:04,703 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-09T03:20:04,704 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-12-09T03:20:04,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-12-09T03:20:04,707 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-12-09T03:20:04,707 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2024-12-09T03:20:04,707 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:04,708 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:04,709 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T03:20:04,709 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-12-09T03:20:04,709 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-12-09T03:20:04,709 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-12-09T03:20:04,712 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:04,712 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:04,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=23, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:04,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-12-09T03:20:04,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=24, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:04,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-12-09T03:20:04,723 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:04,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:04,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:04,727 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:04,728 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=25, ppid=23, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:04,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-12-09T03:20:04,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-12-09T03:20:04,881 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:04,882 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=25 2024-12-09T03:20:04,883 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T03:20:04,883 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T03:20:04,883 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T03:20:04,883 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:04,883 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960018, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:04,884 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960018, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:04,986 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960018, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:04,987 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=25}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3699cd67 to 127.0.0.1:63605 2024-12-09T03:20:04,987 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=25}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:04,988 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T03:20:04,989 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=25}] zookeeper.ZKUtil(111): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:04,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:04,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:04,992 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:05,001 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=25}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:05,001 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=25 2024-12-09T03:20:05,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=25 2024-12-09T03:20:05,007 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:05,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=25, resume processing ppid=23 2024-12-09T03:20:05,011 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=25, ppid=23, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 279 msec 2024-12-09T03:20:05,015 DEBUG [PEWorker-5 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:05,015 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T03:20:05,018 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=23, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 303 msec 2024-12-09T03:20:05,021 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:05,029 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=24, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:05,031 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=24, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=318 msec 2024-12-09T03:20:05,031 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:05,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-12-09T03:20:05,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-12-09T03:20:05,038 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:05,039 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T03:20:05,039 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:05,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=26, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:05,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-12-09T03:20:05,042 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:05,044 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=26, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:05,045 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=26, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-09T03:20:05,045 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:05,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-12-09T03:20:05,148 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:05,150 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=testing utility connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:05,154 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility0x0, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:05,156 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): testing utility-0x10074c9f1960019 connected 2024-12-09T03:20:05,203 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[0] Thread=223 (was 222) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) - Thread LEAK? -, OpenFileDescriptor=452 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=528 (was 495) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9658 (was 9675) 2024-12-09T03:20:05,217 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[0] Thread=223, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=528, ProcessCount=11, AvailableMemoryMB=9658 2024-12-09T03:20:05,219 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:05,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=27, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:05,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-12-09T03:20:05,249 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:05,250 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:05,255 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:05,255 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196001a connected 2024-12-09T03:20:05,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-12-09T03:20:05,358 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196001a, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:05,371 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:05,524 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:05,525 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=28 2024-12-09T03:20:05,526 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T03:20:05,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-12-09T03:20:05,545 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=28}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:05,564 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:05,600 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-12-09T03:20:05,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=28 2024-12-09T03:20:05,605 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:05,608 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=28, resume processing ppid=27 2024-12-09T03:20:05,609 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=28, ppid=27, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 234 msec 2024-12-09T03:20:05,608 INFO [PEWorker-5 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T03:20:05,616 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=27, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 390 msec 2024-12-09T03:20:05,649 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x476fe04b to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4ce3f492 2024-12-09T03:20:05,656 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:05,656 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:05,656 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@2f0d7f02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:05,658 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:05,668 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:05,669 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:05,672 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f196001c connected 2024-12-09T03:20:05,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-12-09T03:20:05,847 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:05,848 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T03:20:05,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=29, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:05,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=29 2024-12-09T03:20:05,872 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:05,874 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:05,879 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:05,880 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196001d connected 2024-12-09T03:20:05,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=29 2024-12-09T03:20:05,996 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196001d, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:06,002 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:06,155 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:06,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=30 2024-12-09T03:20:06,156 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:06,158 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:06,158 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:06,158 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:06,158 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f196001c, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:06,159 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f196001c, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:06,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=29 2024-12-09T03:20:06,261 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f196001c, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:06,261 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=30}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x476fe04b to 127.0.0.1:63605 2024-12-09T03:20:06,261 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=30}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:06,284 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:06,284 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:06,285 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-12-09T03:20:06,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=30 2024-12-09T03:20:06,289 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:06,293 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=30, resume processing ppid=29 2024-12-09T03:20:06,293 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T03:20:06,293 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=30, ppid=29, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 287 msec 2024-12-09T03:20:06,295 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=29, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 445 msec 2024-12-09T03:20:06,323 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x24cb90f0 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5c894a58 2024-12-09T03:20:06,336 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:06,336 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:06,337 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@4a9308dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:06,338 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:06,341 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:06,342 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:06,343 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f196001f connected 2024-12-09T03:20:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=29 2024-12-09T03:20:06,478 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:06,478 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:06,480 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-09T03:20:06,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=31, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:06,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=31 2024-12-09T03:20:06,502 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:06,503 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:06,506 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:06,506 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960020 connected 2024-12-09T03:20:06,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=31 2024-12-09T03:20:06,608 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960020, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:06,614 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:06,767 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:06,768 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=32 2024-12-09T03:20:06,768 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:06,770 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:06,770 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:06,770 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:06,770 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f196001f, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:06,770 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f196001f, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:06,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=31 2024-12-09T03:20:06,873 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f196001f, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:06,873 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=32}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x24cb90f0 to 127.0.0.1:63605 2024-12-09T03:20:06,873 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=32}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:06,890 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:06,890 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:06,890 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=32 2024-12-09T03:20:06,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=32 2024-12-09T03:20:06,897 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:06,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=32, resume processing ppid=31 2024-12-09T03:20:06,901 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-09T03:20:06,901 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=32, ppid=31, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 283 msec 2024-12-09T03:20:06,904 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=31, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 421 msec 2024-12-09T03:20:06,934 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x03b7945c to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@20b51842 2024-12-09T03:20:06,938 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:06,939 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:06,939 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@71a14d75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:06,941 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:06,944 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:06,945 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:06,948 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960022 connected 2024-12-09T03:20:07,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=31 2024-12-09T03:20:07,108 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:07,108 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:07,110 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns1:testNamespacesAndTableCfsConfigConflict1=null},bandwidth=0,serial=false 2024-12-09T03:20:07,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=33, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:07,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-12-09T03:20:07,132 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:07,133 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:07,136 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:07,136 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960023 connected 2024-12-09T03:20:07,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-12-09T03:20:07,239 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960023, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:07,239 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:07,242 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=33, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:07,244 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=33, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=132 msec 2024-12-09T03:20:07,244 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:07,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-12-09T03:20:07,429 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config 2024-12-09T03:20:07,430 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:07,431 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-09T03:20:07,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=34, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-12-09T03:20:07,459 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:07,460 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:07,463 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:07,464 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960024 connected 2024-12-09T03:20:07,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-12-09T03:20:07,566 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960024, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:07,576 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:07,729 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:07,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=35 2024-12-09T03:20:07,731 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:07,734 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:07,734 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:07,734 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:07,734 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960022, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:07,734 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960022, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:07,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-12-09T03:20:07,838 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960022, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:07,838 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=35}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x03b7945c to 127.0.0.1:63605 2024-12-09T03:20:07,838 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=35}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:07,863 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:07,863 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:07,864 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-12-09T03:20:07,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=35 2024-12-09T03:20:07,868 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:07,871 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=35, resume processing ppid=34 2024-12-09T03:20:07,872 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=35, ppid=34, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 292 msec 2024-12-09T03:20:07,872 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-09T03:20:07,876 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 441 msec 2024-12-09T03:20:07,906 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3cf8c953 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@e699ebb 2024-12-09T03:20:07,911 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:07,911 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:07,912 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@5bfa7ea6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:07,913 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:07,917 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:07,918 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:07,919 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960026 connected 2024-12-09T03:20:08,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-12-09T03:20:08,078 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:08,078 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:08,080 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-09T03:20:08,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=36, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:08,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-12-09T03:20:08,101 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:08,101 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:08,105 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:08,107 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960027 connected 2024-12-09T03:20:08,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-12-09T03:20:08,209 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:08,211 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960027, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:08,211 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=36, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:08,213 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=36, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=131 msec 2024-12-09T03:20:08,213 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:08,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-12-09T03:20:08,408 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config 2024-12-09T03:20:08,409 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:08,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=37, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:08,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=37 2024-12-09T03:20:08,414 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:08,417 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:08,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:08,418 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:08,419 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:08,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=37 2024-12-09T03:20:08,572 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:08,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=38 2024-12-09T03:20:08,573 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T03:20:08,573 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T03:20:08,573 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T03:20:08,573 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:08,573 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960026, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:08,574 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960026, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:08,677 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960026, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:08,678 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=38}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3cf8c953 to 127.0.0.1:63605 2024-12-09T03:20:08,678 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=38}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:08,678 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T03:20:08,680 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=38}] zookeeper.ZKUtil(111): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:08,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:08,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:08,683 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:08,683 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=38}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:08,684 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=38 2024-12-09T03:20:08,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=38 2024-12-09T03:20:08,687 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:08,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=38, resume processing ppid=37 2024-12-09T03:20:08,690 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=38, ppid=37, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 268 msec 2024-12-09T03:20:08,694 DEBUG [PEWorker-1 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:08,694 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T03:20:08,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=37, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 285 msec 2024-12-09T03:20:08,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=37 2024-12-09T03:20:08,727 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:08,728 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:08,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=39, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:08,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-12-09T03:20:08,731 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:08,733 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=39, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:08,735 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=39, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-09T03:20:08,735 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:08,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-12-09T03:20:08,839 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T03:20:08,840 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:08,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=40, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:08,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=40 2024-12-09T03:20:08,843 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:08,845 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=40, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:08,848 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=40, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-09T03:20:08,849 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:08,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=40 2024-12-09T03:20:08,948 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:08,964 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[0] Thread=223 (was 223), OpenFileDescriptor=452 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=517 (was 528), ProcessCount=11 (was 11), AvailableMemoryMB=9631 (was 9658) 2024-12-09T03:20:08,974 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAddRemovePeer[0] Thread=223, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=517, ProcessCount=11, AvailableMemoryMB=9631 2024-12-09T03:20:08,975 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:08,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=41, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:08,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41 2024-12-09T03:20:08,996 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:08,997 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:09,000 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:09,001 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960028 connected 2024-12-09T03:20:09,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41 2024-12-09T03:20:09,102 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960028, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:09,110 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:09,264 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:09,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=42 2024-12-09T03:20:09,265 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=42}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T03:20:09,282 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=42}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:09,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41 2024-12-09T03:20:09,300 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=42}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:09,309 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=42}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=42 2024-12-09T03:20:09,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=42 2024-12-09T03:20:09,313 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:09,319 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=42, resume processing ppid=41 2024-12-09T03:20:09,320 INFO [PEWorker-3 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T03:20:09,320 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=42, ppid=41, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 203 msec 2024-12-09T03:20:09,323 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=41, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 345 msec 2024-12-09T03:20:09,348 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x341fc308 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@724402d7 2024-12-09T03:20:09,361 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:09,362 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:09,362 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@4d1e5e10, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:09,363 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:09,384 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:09,388 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:09,388 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f196002a connected 2024-12-09T03:20:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41 2024-12-09T03:20:09,608 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:09,608 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=43, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=43 2024-12-09T03:20:09,634 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:09,635 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:09,641 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:09,642 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196002b connected 2024-12-09T03:20:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=43 2024-12-09T03:20:09,744 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196002b, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:09,744 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:121) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:09,746 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=43, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:121) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:09,748 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=43, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=138 msec 2024-12-09T03:20:09,748 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:09,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=43 2024-12-09T03:20:09,928 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Replication peer 1 already exists 2024-12-09T03:20:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-09T03:20:09,937 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:09,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=44, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:09,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=44 2024-12-09T03:20:09,939 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:09,941 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=44, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:09,943 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=44, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-09T03:20:09,943 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:10,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=44 2024-12-09T03:20:10,048 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:10,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-09T03:20:10,050 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=2, config=clusterKey=127.0.0.1:63605:/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:10,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=45, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:10,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-12-09T03:20:10,093 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:10,096 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:10,110 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:10,110 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196002c connected 2024-12-09T03:20:10,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-12-09T03:20:10,212 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196002c, quorum=127.0.0.1:63605, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:10,220 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:10,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-12-09T03:20:10,373 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:10,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=46 2024-12-09T03:20:10,374 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=2, type=ADD_PEER 2024-12-09T03:20:10,385 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 2000 millis 2024-12-09T03:20:10,399 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=46}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:10,421 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.ReplicationSource(229): queueId=2, ReplicationSource: 2, currentBandwidth=0 2024-12-09T03:20:10,428 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=46 2024-12-09T03:20:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=46 2024-12-09T03:20:10,432 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 2 for ADD on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:10,437 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=46, resume processing ppid=45 2024-12-09T03:20:10,437 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=46, ppid=45, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 213 msec 2024-12-09T03:20:10,437 INFO [PEWorker-3 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 2, config clusterKey=127.0.0.1:63605:/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T03:20:10,440 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=45, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 387 msec 2024-12-09T03:20:10,473 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7e7a4ce1 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2d8f9ee 2024-12-09T03:20:10,478 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,2 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:10,479 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,2 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:10,479 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@4620bb62, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:10,480 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 2 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:10,483 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,2-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 20x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:10,484 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,2 {}] regionserver.ReplicationSource(569): peerId=2, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:10,484 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 2-0x10074c9f196002e connected 2024-12-09T03:20:10,490 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T03:20:10,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-12-09T03:20:10,678 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 2 completed 2024-12-09T03:20:10,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-09T03:20:10,680 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:10,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=47, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:10,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-12-09T03:20:10,685 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:10,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:10,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:10,688 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:10,690 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:10,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-12-09T03:20:10,842 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:10,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=48 2024-12-09T03:20:10,844 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T03:20:10,844 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T03:20:10,844 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T03:20:10,844 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:10,845 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f196002a, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:10,845 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f196002a, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:10,947 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f196002a, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:10,947 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=48}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x341fc308 to 127.0.0.1:63605 2024-12-09T03:20:10,948 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=48}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:10,948 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T03:20:10,950 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=48}] zookeeper.ZKUtil(111): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:10,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:10,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:10,954 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:10,956 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=48}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:10,956 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=48 2024-12-09T03:20:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=48 2024-12-09T03:20:10,959 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:10,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=48, resume processing ppid=47 2024-12-09T03:20:10,962 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=48, ppid=47, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 269 msec 2024-12-09T03:20:10,967 DEBUG [PEWorker-5 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:10,967 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T03:20:10,973 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=47, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 287 msec 2024-12-09T03:20:11,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-12-09T03:20:11,007 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:11,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-09T03:20:11,009 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:11,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=49, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:11,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-12-09T03:20:11,014 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/peer-state 2024-12-09T03:20:11,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/peer-state 2024-12-09T03:20:11,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/2 2024-12-09T03:20:11,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2 2024-12-09T03:20:11,019 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:11,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-12-09T03:20:11,171 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:11,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=50 2024-12-09T03:20:11,172 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=2, type=REMOVE_PEER 2024-12-09T03:20:11,173 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 2: 0 2024-12-09T03:20:11,173 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.ReplicationSource(657): peerId=2, Closing source 2 because: Replication stream was removed by a user 2024-12-09T03:20:11,173 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,2 {}] regionserver.ReplicationSource(508): peerId=2, Interrupted while sleeping between retries 2024-12-09T03:20:11,173 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,2 {}] zookeeper.ZKUtil(165): connection to cluster: 2-0x10074c9f196002e, quorum=127.0.0.1:63605, baseZNode=/hbase-test2 Unable to set watcher on znode (/hbase-test2/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:11,173 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,2 {}] zookeeper.ZKWatcher(734): connection to cluster: 2-0x10074c9f196002e, quorum=127.0.0.1:63605, baseZNode=/hbase-test2 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:11,275 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,2-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 2-0x10074c9f196002e, quorum=127.0.0.1:63605, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:11,276 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=50}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7e7a4ce1 to 127.0.0.1:63605 2024-12-09T03:20:11,276 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=50}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:11,276 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.ReplicationSourceManager(430): Done with the queue 2 2024-12-09T03:20:11,279 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=50}] zookeeper.ZKUtil(111): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/2/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:11,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/2/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:11,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/2 2024-12-09T03:20:11,282 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/2 2024-12-09T03:20:11,283 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=50}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/2 not found in hfile reference queue. 2024-12-09T03:20:11,283 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=50 2024-12-09T03:20:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=50 2024-12-09T03:20:11,286 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 2 for REMOVE on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:11,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=50, resume processing ppid=49 2024-12-09T03:20:11,289 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=50, ppid=49, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 267 msec 2024-12-09T03:20:11,301 DEBUG [PEWorker-1 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/2 not found in hfile reference queue. 2024-12-09T03:20:11,301 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(76): Successfully removed peer 2 2024-12-09T03:20:11,303 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=49, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 292 msec 2024-12-09T03:20:11,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-12-09T03:20:11,327 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 2 completed 2024-12-09T03:20:11,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-09T03:20:11,329 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:11,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=51, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-12-09T03:20:11,332 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:11,334 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=51, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:11,335 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=51, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-09T03:20:11,335 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:11,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-12-09T03:20:11,438 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T03:20:11,439 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:11,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=52, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:11,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=52 2024-12-09T03:20:11,442 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:11,443 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=52, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:11,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=52, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-09T03:20:11,446 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:11,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=52 2024-12-09T03:20:11,548 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:11,570 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAddRemovePeer[0] Thread=219 (was 223), OpenFileDescriptor=446 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=517 (was 517), ProcessCount=11 (was 11), AvailableMemoryMB=9615 (was 9631) 2024-12-09T03:20:11,594 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[0] Thread=219, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=517, ProcessCount=11, AvailableMemoryMB=9614 2024-12-09T03:20:11,597 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=whatever,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:11,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=53, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:11,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53 2024-12-09T03:20:11,601 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:291) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:288) ~[classes/:?] ... 12 more 2024-12-09T03:20:11,603 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=53, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:291) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:288) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:11,605 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=53, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=7 msec 2024-12-09T03:20:11,605 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:11,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53 2024-12-09T03:20:11,708 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Can not instantiate configured replication endpoint class=whatever 2024-12-09T03:20:11,714 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:11,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=54, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:11,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-12-09T03:20:11,717 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:11,719 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=54, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:11,720 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=54, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-09T03:20:11,720 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:11,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-12-09T03:20:11,828 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T03:20:11,828 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:11,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=55, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:11,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-12-09T03:20:11,832 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:11,834 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=55, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:11,836 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=55, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=7 msec 2024-12-09T03:20:11,836 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:11,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-12-09T03:20:11,938 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:11,953 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[0] Thread=219 (was 219), OpenFileDescriptor=446 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=517 (was 517), ProcessCount=11 (was 11), AvailableMemoryMB=9606 (was 9614) 2024-12-09T03:20:11,965 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[0] Thread=219, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=517, ProcessCount=11, AvailableMemoryMB=9606 2024-12-09T03:20:11,966 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=whatever,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:11,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=56, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:11,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=56 2024-12-09T03:20:11,970 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] ... 13 more 2024-12-09T03:20:11,971 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=56, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:11,973 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=56, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=6 msec 2024-12-09T03:20:11,973 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:12,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=56 2024-12-09T03:20:12,078 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Invalid cluster key: whatever 2024-12-09T03:20:12,079 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:12,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=57, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:12,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-12-09T03:20:12,081 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:12,083 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=57, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:12,084 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=57, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-09T03:20:12,084 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:12,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-12-09T03:20:12,188 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T03:20:12,189 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:12,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=58, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:12,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=58 2024-12-09T03:20:12,193 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:12,194 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=58, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:12,195 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=58, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-12-09T03:20:12,195 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:12,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=58 2024-12-09T03:20:12,298 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:12,313 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[0] Thread=219 (was 219), OpenFileDescriptor=444 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=517 (was 517), ProcessCount=11 (was 11), AvailableMemoryMB=9605 (was 9606) 2024-12-09T03:20:12,324 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[0] Thread=219, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=517, ProcessCount=11, AvailableMemoryMB=9605 2024-12-09T03:20:12,325 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:12,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=59, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:12,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-12-09T03:20:12,351 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:12,351 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:12,354 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:12,355 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196002f connected 2024-12-09T03:20:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-12-09T03:20:12,457 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196002f, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:12,464 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:12,618 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:12,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=60 2024-12-09T03:20:12,619 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T03:20:12,640 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=60}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-12-09T03:20:12,657 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:12,666 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=60 2024-12-09T03:20:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=60 2024-12-09T03:20:12,675 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:12,679 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=60, resume processing ppid=59 2024-12-09T03:20:12,679 INFO [PEWorker-2 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T03:20:12,679 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=60, ppid=59, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 211 msec 2024-12-09T03:20:12,682 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=59, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 354 msec 2024-12-09T03:20:12,726 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x391791c7 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@68ed0073 2024-12-09T03:20:12,730 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:12,731 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:12,731 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@6922afb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:12,732 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:12,738 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:12,738 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:12,739 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960031 connected 2024-12-09T03:20:12,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-12-09T03:20:12,958 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:12,959 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T03:20:12,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=61, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:12,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-12-09T03:20:12,991 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:12,992 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:12,998 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:12,998 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960032 connected 2024-12-09T03:20:13,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-12-09T03:20:13,100 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960032, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:13,105 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:13,258 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:13,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=62 2024-12-09T03:20:13,259 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:13,261 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:13,261 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:13,261 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:13,261 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960031, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:13,261 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960031, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:13,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-12-09T03:20:13,364 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960031, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:13,364 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=62}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x391791c7 to 127.0.0.1:63605 2024-12-09T03:20:13,364 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=62}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:13,389 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:13,389 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:13,390 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-12-09T03:20:13,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=62 2024-12-09T03:20:13,394 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:13,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=62, resume processing ppid=61 2024-12-09T03:20:13,399 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T03:20:13,399 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=62, ppid=61, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 289 msec 2024-12-09T03:20:13,402 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=61, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 440 msec 2024-12-09T03:20:13,437 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6e8c3202 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7c29b7d1 2024-12-09T03:20:13,442 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:13,443 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:13,443 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@2d356c87, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:13,445 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:13,460 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:13,460 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960034 connected 2024-12-09T03:20:13,460 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:13,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-12-09T03:20:13,587 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:13,587 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:13,589 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-12-09T03:20:13,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=63, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:13,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-12-09T03:20:13,612 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:13,613 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:13,618 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:13,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-12-09T03:20:13,718 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:13,723 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:13,875 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:13,876 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=64 2024-12-09T03:20:13,877 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:13,878 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:13,878 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:13,878 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:13,879 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960034, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:13,879 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960034, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:13,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-12-09T03:20:13,981 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960034, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:13,981 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=64}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6e8c3202 to 127.0.0.1:63605 2024-12-09T03:20:13,981 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=64}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:14,002 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:14,003 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:14,003 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=64 2024-12-09T03:20:14,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=64 2024-12-09T03:20:14,006 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:14,018 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=64, resume processing ppid=63 2024-12-09T03:20:14,018 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=64, ppid=63, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 283 msec 2024-12-09T03:20:14,018 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-12-09T03:20:14,020 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=63, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 429 msec 2024-12-09T03:20:14,046 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7e2c0889 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@674e11e9 2024-12-09T03:20:14,051 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:14,052 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:14,053 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@45bb665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:14,054 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:14,070 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:14,070 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:14,071 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960037 connected 2024-12-09T03:20:14,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-12-09T03:20:14,218 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:14,218 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:14,220 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:14,221 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-09T03:20:14,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=65, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:14,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-12-09T03:20:14,247 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:14,248 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:14,250 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:14,251 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960038 connected 2024-12-09T03:20:14,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-12-09T03:20:14,353 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960038, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:14,360 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:14,513 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:14,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=66 2024-12-09T03:20:14,514 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:14,515 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:14,516 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:14,516 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:14,516 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960037, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:14,516 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960037, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:14,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-12-09T03:20:14,619 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960037, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:14,619 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=66}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7e2c0889 to 127.0.0.1:63605 2024-12-09T03:20:14,619 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=66}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:14,642 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:14,643 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:14,643 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-12-09T03:20:14,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=66 2024-12-09T03:20:14,646 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:14,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=66, resume processing ppid=65 2024-12-09T03:20:14,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=66, ppid=65, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 287 msec 2024-12-09T03:20:14,649 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-09T03:20:14,652 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=65, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 428 msec 2024-12-09T03:20:14,691 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x490793c8 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@509d3d5e 2024-12-09T03:20:14,696 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:14,697 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:14,697 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@6a75506a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:14,699 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:14,704 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:14,705 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:14,708 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f196003a connected 2024-12-09T03:20:14,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-12-09T03:20:14,847 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:14,848 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:14,849 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:14,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=67, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:14,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67 2024-12-09T03:20:14,858 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:14,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:14,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:14,861 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:14,862 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:14,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67 2024-12-09T03:20:15,016 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:15,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=68 2024-12-09T03:20:15,017 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T03:20:15,017 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T03:20:15,017 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T03:20:15,017 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:15,018 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f196003a, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:15,018 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f196003a, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:15,121 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f196003a, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:15,121 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=68}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x490793c8 to 127.0.0.1:63605 2024-12-09T03:20:15,121 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=68}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:15,122 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T03:20:15,124 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=68}] zookeeper.ZKUtil(111): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:15,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:15,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:15,127 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:15,127 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=68}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:15,127 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=68 2024-12-09T03:20:15,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=68 2024-12-09T03:20:15,130 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:15,133 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=68, resume processing ppid=67 2024-12-09T03:20:15,133 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=68, ppid=67, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 268 msec 2024-12-09T03:20:15,136 DEBUG [PEWorker-1 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:15,136 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T03:20:15,138 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=67, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 287 msec 2024-12-09T03:20:15,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67 2024-12-09T03:20:15,167 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:15,167 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:15,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=69, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:15,169 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:15,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-12-09T03:20:15,171 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=69, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:15,172 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=69, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-09T03:20:15,172 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:15,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-12-09T03:20:15,278 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T03:20:15,278 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:15,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=70, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:15,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=70 2024-12-09T03:20:15,288 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:15,290 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=70, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:15,291 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=70, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=12 msec 2024-12-09T03:20:15,292 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:15,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=70 2024-12-09T03:20:15,387 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:15,401 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[0] Thread=219 (was 219), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=524 (was 517) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9562 (was 9605) 2024-12-09T03:20:15,411 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[0] Thread=219, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=524, ProcessCount=11, AvailableMemoryMB=9561 2024-12-09T03:20:15,412 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:15,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=71, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:15,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-12-09T03:20:15,433 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:15,434 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:15,437 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:15,438 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196003b connected 2024-12-09T03:20:15,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-12-09T03:20:15,545 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196003b, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:15,552 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:15,703 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:15,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=72 2024-12-09T03:20:15,704 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T03:20:15,730 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=72}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:15,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-12-09T03:20:15,755 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:15,781 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=72 2024-12-09T03:20:15,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=72 2024-12-09T03:20:15,789 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:15,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=72, resume processing ppid=71 2024-12-09T03:20:15,796 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=72, ppid=71, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 237 msec 2024-12-09T03:20:15,796 INFO [PEWorker-1 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T03:20:15,804 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=71, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 384 msec 2024-12-09T03:20:15,827 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0b04d94a to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@85c6fb 2024-12-09T03:20:15,855 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:15,856 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:15,856 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@53aaeaad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:15,858 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:15,873 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:15,878 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:15,880 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f196003d connected 2024-12-09T03:20:16,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-12-09T03:20:16,047 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:16,048 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T03:20:16,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=73, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:16,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-12-09T03:20:16,073 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:16,074 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:16,077 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:16,078 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196003e connected 2024-12-09T03:20:16,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-12-09T03:20:16,180 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196003e, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:16,189 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:16,341 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:16,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=74 2024-12-09T03:20:16,342 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:16,344 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:16,344 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:16,344 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:16,344 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f196003d, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:16,345 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f196003d, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:16,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-12-09T03:20:16,447 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f196003d, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:16,447 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=74}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0b04d94a to 127.0.0.1:63605 2024-12-09T03:20:16,447 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=74}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:16,450 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T03:20:16,470 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:16,471 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:16,475 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=74 2024-12-09T03:20:16,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=74 2024-12-09T03:20:16,479 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:16,482 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=74, resume processing ppid=73 2024-12-09T03:20:16,482 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T03:20:16,482 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=74, ppid=73, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 290 msec 2024-12-09T03:20:16,485 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=73, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 434 msec 2024-12-09T03:20:16,530 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x02ddd560 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7f5480f6 2024-12-09T03:20:16,540 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:16,541 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:16,542 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@507e47b7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:16,543 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:16,552 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:16,552 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:16,556 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960040 connected 2024-12-09T03:20:16,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-12-09T03:20:16,677 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:16,678 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:16,679 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-12-09T03:20:16,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=75, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:16,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-12-09T03:20:16,705 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:16,706 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:16,726 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:16,726 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960041 connected 2024-12-09T03:20:16,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-12-09T03:20:16,828 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960041, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:16,834 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:16,987 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:16,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=76 2024-12-09T03:20:16,988 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:16,990 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:16,990 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:16,995 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:16,995 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960040, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:16,995 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960040, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:16,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-12-09T03:20:17,098 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960040, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:17,098 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=76}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x02ddd560 to 127.0.0.1:63605 2024-12-09T03:20:17,098 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=76}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:17,118 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:17,119 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:17,119 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=76 2024-12-09T03:20:17,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=76 2024-12-09T03:20:17,123 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:17,127 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=76, resume processing ppid=75 2024-12-09T03:20:17,127 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=76, ppid=75, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 289 msec 2024-12-09T03:20:17,127 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-12-09T03:20:17,129 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=75, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 448 msec 2024-12-09T03:20:17,159 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x25851c31 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4b2ebf36 2024-12-09T03:20:17,164 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:17,164 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:17,164 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@2d245c4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:17,165 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:17,170 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:17,171 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:17,171 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960043 connected 2024-12-09T03:20:17,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-12-09T03:20:17,308 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:17,308 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:17,309 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:17,311 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-09T03:20:17,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=77, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:17,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-12-09T03:20:17,337 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:17,338 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:17,344 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:17,345 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960044 connected 2024-12-09T03:20:17,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-12-09T03:20:17,447 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960044, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:17,451 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:17,604 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:17,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=78 2024-12-09T03:20:17,605 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:17,606 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:17,606 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:17,606 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:17,607 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960043, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:17,607 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960043, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:17,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-12-09T03:20:17,709 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960043, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:17,710 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=78}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x25851c31 to 127.0.0.1:63605 2024-12-09T03:20:17,710 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=78}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:17,734 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:17,734 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:17,735 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=78 2024-12-09T03:20:17,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=78 2024-12-09T03:20:17,738 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:17,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=78, resume processing ppid=77 2024-12-09T03:20:17,745 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=78, ppid=77, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 287 msec 2024-12-09T03:20:17,745 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-09T03:20:17,753 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=77, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 434 msec 2024-12-09T03:20:17,785 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x00d48887 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2e44f495 2024-12-09T03:20:17,791 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:17,791 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:17,792 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@2d00f109, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:17,792 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:17,796 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:17,796 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:17,797 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960046 connected 2024-12-09T03:20:17,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-12-09T03:20:17,938 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:17,938 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:17,941 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:17,942 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-09T03:20:17,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=79, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:17,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-12-09T03:20:17,967 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:17,968 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:17,972 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:17,973 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960047 connected 2024-12-09T03:20:18,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-12-09T03:20:18,078 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960047, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:18,082 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:18,234 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:18,235 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=80 2024-12-09T03:20:18,235 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:18,237 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:18,237 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:18,237 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:18,237 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960046, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:18,238 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960046, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:18,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-12-09T03:20:18,341 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960046, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:18,341 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=80}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x00d48887 to 127.0.0.1:63605 2024-12-09T03:20:18,341 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=80}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:18,360 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:18,360 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:18,360 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=80 2024-12-09T03:20:18,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=80 2024-12-09T03:20:18,364 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:18,367 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=80, resume processing ppid=79 2024-12-09T03:20:18,367 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=80, ppid=79, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 282 msec 2024-12-09T03:20:18,367 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-09T03:20:18,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=79, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 425 msec 2024-12-09T03:20:18,401 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x66d5b904 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7ccdedb4 2024-12-09T03:20:18,406 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:18,406 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:18,406 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@668ebca2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:18,407 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:18,424 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:18,424 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:18,440 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960049 connected 2024-12-09T03:20:18,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-12-09T03:20:18,568 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:18,569 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:18,571 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:18,572 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-09T03:20:18,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=81, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:18,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-12-09T03:20:18,598 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:18,598 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:18,604 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:18,608 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196004a connected 2024-12-09T03:20:18,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-12-09T03:20:18,710 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196004a, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:18,716 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:18,868 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:18,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=82 2024-12-09T03:20:18,869 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:18,870 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:18,870 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:18,870 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:18,870 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960049, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:18,871 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960049, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:18,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-12-09T03:20:18,973 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960049, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:18,973 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=82}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x66d5b904 to 127.0.0.1:63605 2024-12-09T03:20:18,973 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=82}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:18,989 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:18,989 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:18,989 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=82 2024-12-09T03:20:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=82 2024-12-09T03:20:18,992 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:18,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=82, resume processing ppid=81 2024-12-09T03:20:18,994 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-09T03:20:18,994 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=82, ppid=81, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 276 msec 2024-12-09T03:20:18,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=81, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 422 msec 2024-12-09T03:20:19,024 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x39a23b42 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@a263725 2024-12-09T03:20:19,030 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:19,031 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:19,031 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@5c587291, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:19,032 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:19,038 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:19,039 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:19,039 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f196004c connected 2024-12-09T03:20:19,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-12-09T03:20:19,208 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:19,208 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:19,209 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:19,210 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-09T03:20:19,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=83, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:19,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-12-09T03:20:19,227 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:19,227 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:19,233 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:19,234 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196004d connected 2024-12-09T03:20:19,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-12-09T03:20:19,335 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196004d, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:19,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:19,496 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:19,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=84 2024-12-09T03:20:19,497 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:19,499 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:19,499 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:19,499 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:19,499 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f196004c, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:19,499 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f196004c, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:19,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-12-09T03:20:19,604 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f196004c, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:19,605 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=84}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x39a23b42 to 127.0.0.1:63605 2024-12-09T03:20:19,605 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=84}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:19,628 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:19,629 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:19,629 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=84 2024-12-09T03:20:19,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=84 2024-12-09T03:20:19,632 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:19,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=84, resume processing ppid=83 2024-12-09T03:20:19,634 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-09T03:20:19,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=84, ppid=83, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 288 msec 2024-12-09T03:20:19,636 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=83, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 424 msec 2024-12-09T03:20:19,661 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5c06e288 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@249067c9 2024-12-09T03:20:19,665 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:19,666 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:19,666 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@17bf9383, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:19,667 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:19,669 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:19,670 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:19,670 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f196004f connected 2024-12-09T03:20:19,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-12-09T03:20:19,837 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:19,838 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:19,839 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-09T03:20:19,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=85, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:19,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-12-09T03:20:19,857 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:19,857 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:19,860 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:19,860 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960050 connected 2024-12-09T03:20:19,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-12-09T03:20:19,962 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960050, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:19,966 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:20,118 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:20,119 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=86 2024-12-09T03:20:20,119 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:20,120 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=86 2024-12-09T03:20:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=86 2024-12-09T03:20:20,123 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:20,124 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=86, resume processing ppid=85 2024-12-09T03:20:20,124 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=86, ppid=85, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 157 msec 2024-12-09T03:20:20,124 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-09T03:20:20,126 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=85, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 285 msec 2024-12-09T03:20:20,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-12-09T03:20:20,157 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:20,157 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:20,158 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:20,159 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-12-09T03:20:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=87, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:20,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-12-09T03:20:20,176 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:20,177 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:20,181 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:20,181 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960051 connected 2024-12-09T03:20:20,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-12-09T03:20:20,284 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960051, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:20,288 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:20,440 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:20,442 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=88 2024-12-09T03:20:20,442 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:20,443 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:20,443 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:20,443 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:20,443 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f196004f, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:20,444 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f196004f, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:20,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-12-09T03:20:20,546 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f196004f, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:20,546 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=88}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5c06e288 to 127.0.0.1:63605 2024-12-09T03:20:20,546 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=88}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:20,562 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:20,562 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:20,562 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-12-09T03:20:20,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=88 2024-12-09T03:20:20,565 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:20,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=88, resume processing ppid=87 2024-12-09T03:20:20,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=88, ppid=87, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 277 msec 2024-12-09T03:20:20,567 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-12-09T03:20:20,568 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=87, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 408 msec 2024-12-09T03:20:20,593 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6ab805f9 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@60959f36 2024-12-09T03:20:20,596 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:20,597 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:20,597 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@2ec1e61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:20,598 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:20,600 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:20,601 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:20,601 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960053 connected 2024-12-09T03:20:20,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-12-09T03:20:20,787 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:20,788 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:20,789 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-12-09T03:20:20,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=89, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:20,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-12-09T03:20:20,807 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:20,807 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:20,810 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:20,810 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960054 connected 2024-12-09T03:20:20,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-12-09T03:20:20,911 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960054, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:20,918 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:21,070 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:21,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=90 2024-12-09T03:20:21,070 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:21,072 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:21,072 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:21,072 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:21,072 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960053, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:21,072 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960053, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:21,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-12-09T03:20:21,174 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960053, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:21,175 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=90}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6ab805f9 to 127.0.0.1:63605 2024-12-09T03:20:21,175 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=90}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:21,190 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:21,191 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:21,191 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-12-09T03:20:21,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=90 2024-12-09T03:20:21,193 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:21,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=90, resume processing ppid=89 2024-12-09T03:20:21,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=90, ppid=89, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 275 msec 2024-12-09T03:20:21,195 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-12-09T03:20:21,197 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=89, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 406 msec 2024-12-09T03:20:21,221 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4e9db348 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@9176d70 2024-12-09T03:20:21,225 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:21,226 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:21,226 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@49cc53b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:21,227 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:21,229 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:21,229 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:21,229 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960056 connected 2024-12-09T03:20:21,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-12-09T03:20:21,417 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:21,417 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:21,418 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:21,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=91, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:21,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91 2024-12-09T03:20:21,422 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:21,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:21,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:21,424 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:21,425 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:21,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91 2024-12-09T03:20:21,577 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:21,578 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=92 2024-12-09T03:20:21,578 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T03:20:21,578 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T03:20:21,578 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T03:20:21,578 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:21,579 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960056, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:21,579 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960056, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:21,681 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960056, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:21,681 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=92}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4e9db348 to 127.0.0.1:63605 2024-12-09T03:20:21,681 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=92}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:21,681 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T03:20:21,683 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=92}] zookeeper.ZKUtil(111): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:21,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:21,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:21,685 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:21,686 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=92}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:21,686 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-12-09T03:20:21,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=92 2024-12-09T03:20:21,689 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:21,691 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=92, resume processing ppid=91 2024-12-09T03:20:21,691 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=92, ppid=91, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 264 msec 2024-12-09T03:20:21,694 DEBUG [PEWorker-1 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:21,694 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T03:20:21,696 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=91, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 276 msec 2024-12-09T03:20:21,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91 2024-12-09T03:20:21,737 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:21,738 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:21,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=93, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=93 2024-12-09T03:20:21,740 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:21,741 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=93, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:21,742 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=93, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-09T03:20:21,743 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:21,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=93 2024-12-09T03:20:21,848 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T03:20:21,848 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:21,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=94, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:21,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=94 2024-12-09T03:20:21,850 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:21,851 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=94, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:21,852 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=94, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T03:20:21,852 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=94 2024-12-09T03:20:21,957 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:21,969 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[0] Thread=219 (was 219), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=562 (was 524) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9451 (was 9561) 2024-12-09T03:20:21,977 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[0] Thread=219, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=562, ProcessCount=11, AvailableMemoryMB=9451 2024-12-09T03:20:21,978 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=95, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=95 2024-12-09T03:20:21,995 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:21,996 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:21,998 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:21,998 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960057 connected 2024-12-09T03:20:22,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=95 2024-12-09T03:20:22,100 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960057, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:22,106 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:22,257 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:22,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=96 2024-12-09T03:20:22,258 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T03:20:22,275 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=96}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:22,290 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:22,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=95 2024-12-09T03:20:22,297 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-12-09T03:20:22,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=96 2024-12-09T03:20:22,299 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:22,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=96, resume processing ppid=95 2024-12-09T03:20:22,301 INFO [PEWorker-5 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T03:20:22,301 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=96, ppid=95, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 193 msec 2024-12-09T03:20:22,302 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=95, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 323 msec 2024-12-09T03:20:22,326 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0bd10e58 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@40f57838 2024-12-09T03:20:22,329 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:22,329 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:22,329 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@639b8371, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:22,330 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:22,332 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:22,333 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:22,333 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960059 connected 2024-12-09T03:20:22,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=95 2024-12-09T03:20:22,607 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-09T03:20:22,611 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3938): Client=jenkins//172.17.0.2 disable replication peer, id=1 2024-12-09T03:20:22,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=97, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure 2024-12-09T03:20:22,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=97 2024-12-09T03:20:22,617 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:22,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=97 2024-12-09T03:20:22,768 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:22,769 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=98 2024-12-09T03:20:22,769 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=DISABLE_PEER 2024-12-09T03:20:22,770 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:22,770 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:22,770 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:22,770 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960059, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:22,771 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960059, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:22,872 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960059, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:22,873 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=98}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0bd10e58 to 127.0.0.1:63605 2024-12-09T03:20:22,873 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=98}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:22,888 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:22,888 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:22,889 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-12-09T03:20:22,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=98 2024-12-09T03:20:22,891 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for DISABLE on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:22,892 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=98, resume processing ppid=97 2024-12-09T03:20:22,892 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=98, ppid=97, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 274 msec 2024-12-09T03:20:22,892 INFO [PEWorker-1 {}] replication.DisablePeerProcedure(67): Successfully disabled peer 1 2024-12-09T03:20:22,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=97, state=SUCCESS; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure in 281 msec 2024-12-09T03:20:22,920 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1b63645d to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@45e58683 2024-12-09T03:20:22,923 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:22,923 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:22,924 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@14895da7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:22,925 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:22,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=97 2024-12-09T03:20:22,927 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: DISABLE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-09T03:20:22,927 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:22,928 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:22,928 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f196005b connected 2024-12-09T03:20:22,928 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:22,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=99, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:22,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=99 2024-12-09T03:20:22,932 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:22,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:22,934 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:22,935 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:22,935 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:23,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=99 2024-12-09T03:20:23,087 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:23,088 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=100 2024-12-09T03:20:23,088 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T03:20:23,088 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T03:20:23,088 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T03:20:23,089 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:23,089 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f196005b, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:23,089 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f196005b, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:23,191 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f196005b, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:23,191 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=100}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1b63645d to 127.0.0.1:63605 2024-12-09T03:20:23,191 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=100}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:23,191 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T03:20:23,193 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=100}] zookeeper.ZKUtil(111): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:23,195 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T03:20:23,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:23,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:23,195 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:23,196 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=100}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:23,196 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-12-09T03:20:23,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=100 2024-12-09T03:20:23,198 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:23,200 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=100, resume processing ppid=99 2024-12-09T03:20:23,200 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=100, ppid=99, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 263 msec 2024-12-09T03:20:23,203 DEBUG [PEWorker-3 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:23,203 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T03:20:23,204 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=99, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 275 msec 2024-12-09T03:20:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=99 2024-12-09T03:20:23,247 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:23,247 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:23,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=101, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=101 2024-12-09T03:20:23,249 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:23,250 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=101, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:23,251 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=101, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T03:20:23,251 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:23,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=101 2024-12-09T03:20:23,358 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T03:20:23,358 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:23,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=102, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:23,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=102 2024-12-09T03:20:23,360 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:23,361 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=102, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:23,362 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=102, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T03:20:23,362 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:23,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=102 2024-12-09T03:20:23,467 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:23,479 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[0] Thread=219 (was 219), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=525 (was 562), ProcessCount=11 (was 11), AvailableMemoryMB=9434 (was 9451) 2024-12-09T03:20:23,487 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[0] Thread=219, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=525, ProcessCount=11, AvailableMemoryMB=9434 2024-12-09T03:20:23,488 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=dummy_peer 2024-12-09T03:20:23,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=103, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:23,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=103 2024-12-09T03:20:23,490 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer dummy_peer, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:23,491 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=103, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:23,491 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=103, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T03:20:23,491 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:23,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=103 2024-12-09T03:20:23,597 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: dummy_peer failed with dummy_peer 2024-12-09T03:20:23,597 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:23,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=104, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:23,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=104 2024-12-09T03:20:23,599 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:23,600 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=104, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:23,601 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=104, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T03:20:23,601 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:23,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=104 2024-12-09T03:20:23,708 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T03:20:23,708 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:23,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=105, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:23,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=105 2024-12-09T03:20:23,710 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:23,711 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=105, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:23,712 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=105, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T03:20:23,712 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:23,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=105 2024-12-09T03:20:23,818 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:23,829 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[0] Thread=221 (was 219) - Thread LEAK? -, OpenFileDescriptor=446 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=525 (was 525), ProcessCount=11 (was 11), AvailableMemoryMB=9432 (was 9434) 2024-12-09T03:20:23,839 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerConfig[0] Thread=221, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=525, ProcessCount=11, AvailableMemoryMB=9432 2024-12-09T03:20:23,840 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:23,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=106, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:23,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-12-09T03:20:23,857 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:23,857 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:23,860 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:23,860 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196005c connected 2024-12-09T03:20:23,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-12-09T03:20:23,962 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196005c, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:23,967 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:24,119 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:24,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=107 2024-12-09T03:20:24,120 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T03:20:24,144 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=107}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:24,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-12-09T03:20:24,164 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:24,174 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=107 2024-12-09T03:20:24,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=107 2024-12-09T03:20:24,176 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:24,178 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=107, resume processing ppid=106 2024-12-09T03:20:24,178 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=107, ppid=106, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 209 msec 2024-12-09T03:20:24,178 INFO [PEWorker-4 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T03:20:24,180 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=106, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 338 msec 2024-12-09T03:20:24,212 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2903fb12 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@525f3471 2024-12-09T03:20:24,216 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:24,216 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:24,216 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@65a7391a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:24,218 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:24,221 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:24,221 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:24,221 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f196005e connected 2024-12-09T03:20:24,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-12-09T03:20:24,467 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:24,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-09T03:20:24,468 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:24,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=108, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:24,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-12-09T03:20:24,471 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:24,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:24,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:24,473 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:24,474 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-12-09T03:20:24,625 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:24,626 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=109 2024-12-09T03:20:24,626 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T03:20:24,626 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T03:20:24,626 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T03:20:24,627 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:24,627 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f196005e, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:24,627 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f196005e, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:24,729 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f196005e, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:24,729 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=109}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2903fb12 to 127.0.0.1:63605 2024-12-09T03:20:24,729 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=109}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:24,729 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T03:20:24,731 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=109}] zookeeper.ZKUtil(111): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:24,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:24,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:24,733 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:24,733 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=109}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:24,733 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=109 2024-12-09T03:20:24,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=109 2024-12-09T03:20:24,736 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:24,737 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=109, resume processing ppid=108 2024-12-09T03:20:24,737 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=109, ppid=108, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 262 msec 2024-12-09T03:20:24,739 DEBUG [PEWorker-2 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:24,739 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T03:20:24,741 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=108, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 272 msec 2024-12-09T03:20:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-12-09T03:20:24,787 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:24,788 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=110, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:24,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-12-09T03:20:24,789 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:24,790 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=110, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:24,791 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=110, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T03:20:24,791 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:24,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-12-09T03:20:24,897 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T03:20:24,898 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:24,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=111, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:24,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=111 2024-12-09T03:20:24,899 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:24,900 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=111, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:24,901 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=111, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T03:20:24,901 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:25,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=111 2024-12-09T03:20:25,007 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:25,021 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerConfig[0] Thread=221 (was 221), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=525 (was 525), ProcessCount=11 (was 11), AvailableMemoryMB=9413 (was 9432) 2024-12-09T03:20:25,030 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[0] Thread=221, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=525, ProcessCount=11, AvailableMemoryMB=9412 2024-12-09T03:20:25,037 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:25,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-09T03:20:25,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] ipc.CallRunner(138): callId: 341 service: MasterService methodName: AddReplicationPeer size: 80 connection: 172.17.0.2:39754 deadline: 1733714485037, exception=java.io.IOException: Replication peer modification disabled 2024-12-09T03:20:25,063 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(168): Call to master failed, tries = 1, maxAttempts = 2, timeout = 120000 ms, time elapsed = 25 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:245) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:431) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 33 more 2024-12-09T03:20:25,167 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:25,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-09T03:20:25,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] ipc.CallRunner(138): callId: 342 service: MasterService methodName: AddReplicationPeer size: 80 connection: 172.17.0.2:39754 deadline: 1733714485167, exception=java.io.IOException: Replication peer modification disabled 2024-12-09T03:20:25,168 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(168): Call to master failed, tries = 2, maxAttempts = 2, timeout = 120000 ms, time elapsed = 131 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:245) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:431) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 33 more 2024-12-09T03:20:25,169 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Failed after attempts=2, exceptions: 2024-12-09T03:20:25.064Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) 2024-12-09T03:20:25.169Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) 2024-12-09T03:20:25,171 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:25,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=112, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:25,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-12-09T03:20:25,173 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:25,174 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=112, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:25,174 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=112, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T03:20:25,174 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:25,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-12-09T03:20:25,277 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T03:20:25,278 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:25,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=113, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:25,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=113 2024-12-09T03:20:25,280 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:25,280 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=113, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:25,281 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=113, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T03:20:25,281 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:25,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=113 2024-12-09T03:20:25,387 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:25,399 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[0] Thread=221 (was 221), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=525 (was 525), ProcessCount=11 (was 11), AvailableMemoryMB=9407 (was 9412) 2024-12-09T03:20:25,407 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[0] Thread=221, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=525, ProcessCount=11, AvailableMemoryMB=9407 2024-12-09T03:20:25,407 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:25,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=114, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:25,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-12-09T03:20:25,425 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:25,425 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:25,428 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:25,428 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196005f connected 2024-12-09T03:20:25,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-12-09T03:20:25,530 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196005f, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:25,535 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:25,686 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:25,687 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=115 2024-12-09T03:20:25,687 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T03:20:25,703 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=115}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:25,718 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:25,725 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=115 2024-12-09T03:20:25,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=115 2024-12-09T03:20:25,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-12-09T03:20:25,728 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:25,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=115, resume processing ppid=114 2024-12-09T03:20:25,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=115, ppid=114, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 193 msec 2024-12-09T03:20:25,730 INFO [PEWorker-1 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T03:20:25,731 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=114, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 322 msec 2024-12-09T03:20:25,757 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0883b862 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@461eb926 2024-12-09T03:20:25,761 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:25,761 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:25,762 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@1b9a1f13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:25,762 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:25,765 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:25,765 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:25,765 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960061 connected 2024-12-09T03:20:26,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-12-09T03:20:26,037 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:26,037 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:26,038 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:26,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=116, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:26,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116 2024-12-09T03:20:26,047 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:26,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:26,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:26,052 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:26,053 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:26,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116 2024-12-09T03:20:26,205 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:26,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=117 2024-12-09T03:20:26,206 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T03:20:26,206 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T03:20:26,206 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T03:20:26,206 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:26,206 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960061, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:26,207 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960061, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:26,309 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960061, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:26,309 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=117}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0883b862 to 127.0.0.1:63605 2024-12-09T03:20:26,309 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=117}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:26,309 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T03:20:26,311 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=117}] zookeeper.ZKUtil(111): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:26,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:26,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:26,314 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:26,314 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=117}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:26,314 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=117 2024-12-09T03:20:26,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=117 2024-12-09T03:20:26,317 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:26,319 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=117, resume processing ppid=116 2024-12-09T03:20:26,319 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=117, ppid=116, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 264 msec 2024-12-09T03:20:26,323 DEBUG [PEWorker-3 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:26,323 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T03:20:26,324 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=116, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 285 msec 2024-12-09T03:20:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116 2024-12-09T03:20:26,357 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:26,357 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:26,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=118, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:26,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=118 2024-12-09T03:20:26,362 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:26,363 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=118, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:26,364 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=118, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec 2024-12-09T03:20:26,364 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:26,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=118 2024-12-09T03:20:26,467 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:26,489 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[0] Thread=221 (was 221), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=525 (was 525), ProcessCount=11 (was 11), AvailableMemoryMB=9379 (was 9407) 2024-12-09T03:20:26,502 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerBandwidth[0] Thread=221, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=525, ProcessCount=11, AvailableMemoryMB=9378 2024-12-09T03:20:26,504 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=119, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:26,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119 2024-12-09T03:20:26,531 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:26,532 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:26,551 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:26,560 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960062 connected 2024-12-09T03:20:26,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119 2024-12-09T03:20:26,663 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960062, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:26,670 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:26,822 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:26,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=120 2024-12-09T03:20:26,824 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T03:20:26,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119 2024-12-09T03:20:26,859 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=120}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:26,882 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:26,892 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-12-09T03:20:26,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=120 2024-12-09T03:20:26,894 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:26,896 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=120, resume processing ppid=119 2024-12-09T03:20:26,896 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=120, ppid=119, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 225 msec 2024-12-09T03:20:26,896 INFO [PEWorker-4 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T03:20:26,898 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=119, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 393 msec 2024-12-09T03:20:26,929 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x080a93a7 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@22e322bd 2024-12-09T03:20:26,934 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:26,934 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:26,934 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@6f995cd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:26,935 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:26,938 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:26,939 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960064 connected 2024-12-09T03:20:26,939 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:27,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119 2024-12-09T03:20:27,138 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:27,138 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:27,139 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-12-09T03:20:27,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=121, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:27,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=121 2024-12-09T03:20:27,166 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:27,166 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:27,171 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:27,219 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T03:20:27,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=121 2024-12-09T03:20:27,272 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:27,276 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:27,428 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:27,429 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=122 2024-12-09T03:20:27,429 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:27,431 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-12-09T03:20:27,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=122 2024-12-09T03:20:27,433 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:27,436 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=122, resume processing ppid=121 2024-12-09T03:20:27,436 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-12-09T03:20:27,436 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=122, ppid=121, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 157 msec 2024-12-09T03:20:27,437 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=121, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 297 msec 2024-12-09T03:20:27,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=121 2024-12-09T03:20:27,457 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:27,457 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:27,458 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:27,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=123, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:27,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=123 2024-12-09T03:20:27,462 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:27,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:27,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:27,465 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:27,466 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:27,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=123 2024-12-09T03:20:27,617 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:27,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=124 2024-12-09T03:20:27,618 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T03:20:27,618 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T03:20:27,618 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T03:20:27,618 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:27,618 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960064, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:27,619 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960064, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:27,721 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960064, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:27,721 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=124}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x080a93a7 to 127.0.0.1:63605 2024-12-09T03:20:27,721 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=124}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:27,721 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T03:20:27,723 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=124}] zookeeper.ZKUtil(111): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:27,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:27,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:27,726 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:27,727 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=124}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:27,727 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-12-09T03:20:27,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=124 2024-12-09T03:20:27,738 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:27,740 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=124, resume processing ppid=123 2024-12-09T03:20:27,740 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=124, ppid=123, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 272 msec 2024-12-09T03:20:27,744 DEBUG [PEWorker-5 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:27,744 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T03:20:27,745 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=123, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 286 msec 2024-12-09T03:20:27,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=123 2024-12-09T03:20:27,777 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:27,777 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:27,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=125, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:27,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=125 2024-12-09T03:20:27,779 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:27,780 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=125, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:27,785 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=125, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-09T03:20:27,785 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:27,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=125 2024-12-09T03:20:27,887 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T03:20:27,888 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:27,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=126, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:27,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=126 2024-12-09T03:20:27,890 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:27,891 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=126, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:27,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=126, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T03:20:27,892 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:27,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=126 2024-12-09T03:20:27,998 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:28,021 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerBandwidth[0] Thread=223 (was 221) - Thread LEAK? -, OpenFileDescriptor=448 (was 446) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=525 (was 525), ProcessCount=11 (was 11), AvailableMemoryMB=9354 (was 9378) 2024-12-09T03:20:28,034 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[0] Thread=223, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=525, ProcessCount=11, AvailableMemoryMB=9354 2024-12-09T03:20:28,036 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:28,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=127, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:28,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=127 2024-12-09T03:20:28,049 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:28,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=127 2024-12-09T03:20:28,201 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:28,202 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=128 2024-12-09T03:20:28,202 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=128}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T03:20:28,249 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=128}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:28,256 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-12-09T03:20:28,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=128 2024-12-09T03:20:28,257 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(587): peerId=1, queueId=1 (queues=1) is replicating from cluster=3b7d4240-16c9-429e-aa98-985bf779f27b to cluster=3b7d4240-16c9-429e-aa98-985bf779f27b 2024-12-09T03:20:28,259 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:28,261 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=128, resume processing ppid=127 2024-12-09T03:20:28,261 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=128, ppid=127, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 210 msec 2024-12-09T03:20:28,261 INFO [PEWorker-3 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T03:20:28,261 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(343): peerId=1, starting shipping worker for walGroupId=66b6cc25935e%2C45493%2C1733714394961 2024-12-09T03:20:28,264 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=127, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 225 msec 2024-12-09T03:20:28,265 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSourceWALReader(112): peerClusterZnode=1, ReplicationSourceWALReaderThread : 1 inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-12-09T03:20:28,268 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1.replicationSource.shipper66b6cc25935e%2C45493%2C1733714394961,1 {}] regionserver.ReplicationSourceShipper(98): Running ReplicationSourceShipper Thread for wal group: 66b6cc25935e%2C45493%2C1733714394961 2024-12-09T03:20:28,268 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1.replicationSource.wal-reader.66b6cc25935e%2C45493%2C1733714394961,1 {}] regionserver.WALEntryStream(260): Creating new reader hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/WALs/66b6cc25935e,45493,1733714394961/66b6cc25935e%2C45493%2C1733714394961.1733714397367, startPosition=0, beingWritten=true 2024-12-09T03:20:28,349 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1.replicationSource.wal-reader.66b6cc25935e%2C45493%2C1733714394961,1 {}] regionserver.ReplicationSourceWALReader(181): Read 0 WAL entries eligible for replication 2024-12-09T03:20:28,349 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1.replicationSource.wal-reader.66b6cc25935e%2C45493%2C1733714394961,1 {}] regionserver.WALEntryStream(223): Reset reader hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/WALs/66b6cc25935e,45493,1733714394961/66b6cc25935e%2C45493%2C1733714394961.1733714397367 to pos 589, reset compression=false 2024-12-09T03:20:28,350 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1.replicationSource.shipper66b6cc25935e%2C45493%2C1733714394961,1 {}] regionserver.ReplicationSourceShipper(110): Shipper from source 1 got entry batch from reader: WALEntryBatch [walEntries=[], lastWalPath=hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/WALs/66b6cc25935e,45493,1733714394961/66b6cc25935e%2C45493%2C1733714394961.1733714397367, lastWalPosition=589, nbRowKeys=0, nbHFiles=0, heapSize=0, lastSeqIds={}, endOfFile=false,usedBufferSize=0] 2024-12-09T03:20:28,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=127 2024-12-09T03:20:28,357 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:28,358 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=2, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:28,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=129, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:28,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=129 2024-12-09T03:20:28,361 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] ... 13 more 2024-12-09T03:20:28,362 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=129, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:28,363 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=129, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=4 msec 2024-12-09T03:20:28,363 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:28,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=129 2024-12-09T03:20:28,468 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 2 failed with Invalid cluster key: 2024-12-09T03:20:28,469 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:28,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=130, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:28,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-12-09T03:20:28,474 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:28,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:28,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:28,476 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:28,477 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:28,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-12-09T03:20:28,630 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:28,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=131 2024-12-09T03:20:28,630 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T03:20:28,631 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T03:20:28,631 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T03:20:28,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-12-09T03:20:29,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-12-09T03:20:29,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-12-09T03:20:29,631 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1.replicationSource.wal-reader.66b6cc25935e%2C45493%2C1733714394961,1 {}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:131) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:166) ~[classes/:?] 2024-12-09T03:20:29,631 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1.replicationSource.shipper66b6cc25935e%2C45493%2C1733714394961,1 {}] regionserver.ReplicationSourceShipper(123): Interrupted while waiting for next replication entry batch java.lang.InterruptedException: null at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1681) ~[?:?] at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) ~[?:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.poll(ReplicationSourceWALReader.java:313) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceShipper.run(ReplicationSourceShipper.java:109) ~[classes/:?] 2024-12-09T03:20:30,034 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-12-09T03:20:30,036 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:42930, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-12-09T03:20:30,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-12-09T03:20:30,631 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.ReplicationSource(711): peerId=1, ReplicationSourceWorker RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1.replicationSource.shipper66b6cc25935e%2C45493%2C1733714394961,1 terminated 2024-12-09T03:20:30,631 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T03:20:30,633 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=131}] zookeeper.ZKUtil(111): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:30,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:30,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:30,640 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:30,640 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=131}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:30,640 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=131 2024-12-09T03:20:30,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=131 2024-12-09T03:20:30,644 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:30,646 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=131, resume processing ppid=130 2024-12-09T03:20:30,646 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=131, ppid=130, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 2.1670 sec 2024-12-09T03:20:30,649 DEBUG [PEWorker-3 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:30,649 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T03:20:30,650 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=130, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 2.1800 sec 2024-12-09T03:20:32,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-12-09T03:20:32,628 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:32,628 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:32,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=132, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:32,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-12-09T03:20:32,631 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:32,632 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=132, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:32,633 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=132, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-09T03:20:32,633 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:32,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-12-09T03:20:32,738 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:32,752 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[0] Thread=222 (was 223), OpenFileDescriptor=453 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=523 (was 525), ProcessCount=11 (was 11), AvailableMemoryMB=9332 (was 9354) 2024-12-09T03:20:32,766 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[1] Thread=222, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=523, ProcessCount=11, AvailableMemoryMB=9330 2024-12-09T03:20:32,768 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:32,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=133, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:32,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=133 2024-12-09T03:20:32,793 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:32,794 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:32,808 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:32,811 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960066 connected 2024-12-09T03:20:32,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=133 2024-12-09T03:20:32,914 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960066, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:32,921 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:33,073 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:33,073 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=134 2024-12-09T03:20:33,074 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T03:20:33,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=133 2024-12-09T03:20:33,093 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=134}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:33,109 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:33,116 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-12-09T03:20:33,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=134 2024-12-09T03:20:33,120 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:33,123 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=134, resume processing ppid=133 2024-12-09T03:20:33,123 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=134, ppid=133, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 199 msec 2024-12-09T03:20:33,123 INFO [PEWorker-5 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T03:20:33,124 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=133, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 355 msec 2024-12-09T03:20:33,148 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0dac1208 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4b90e290 2024-12-09T03:20:33,160 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:33,160 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:33,160 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@74b1e23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:33,161 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:33,164 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:33,164 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:33,172 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960068 connected 2024-12-09T03:20:33,223 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T03:20:33,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=133 2024-12-09T03:20:33,397 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:33,398 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T03:20:33,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=135, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:33,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=135 2024-12-09T03:20:33,416 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:33,417 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:33,423 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:33,423 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960069 connected 2024-12-09T03:20:33,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=135 2024-12-09T03:20:33,525 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960069, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:33,528 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:33,680 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:33,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=136 2024-12-09T03:20:33,681 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:33,683 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:33,683 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:33,683 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:33,683 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960068, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:33,683 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960068, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:33,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=135 2024-12-09T03:20:33,785 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=136}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0dac1208 to 127.0.0.1:63605 2024-12-09T03:20:33,786 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=136}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:33,788 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960068, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:33,803 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:33,803 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:33,806 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-12-09T03:20:33,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=136 2024-12-09T03:20:33,811 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:33,813 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=136, resume processing ppid=135 2024-12-09T03:20:33,813 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=136, ppid=135, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 283 msec 2024-12-09T03:20:33,813 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T03:20:33,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=135, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 415 msec 2024-12-09T03:20:33,842 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x20c59d9e to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@55b103c1 2024-12-09T03:20:33,846 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:33,847 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:33,847 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@1fdca5b4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:33,848 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:33,854 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:33,854 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:33,854 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f196006b connected 2024-12-09T03:20:34,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=135 2024-12-09T03:20:34,027 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:34,028 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:34,029 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:34,029 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:34,030 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-09T03:20:34,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=137, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:34,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=137 2024-12-09T03:20:34,056 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:34,056 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:34,059 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:34,060 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196006c connected 2024-12-09T03:20:34,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=137 2024-12-09T03:20:34,165 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196006c, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:34,169 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:34,320 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:34,321 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=138 2024-12-09T03:20:34,321 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:34,323 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:34,323 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:34,323 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:34,323 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f196006b, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:34,323 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f196006b, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:34,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=137 2024-12-09T03:20:34,426 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f196006b, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:34,426 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=138}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x20c59d9e to 127.0.0.1:63605 2024-12-09T03:20:34,426 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=138}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:34,450 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:34,450 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:34,450 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-12-09T03:20:34,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=138 2024-12-09T03:20:34,454 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:34,456 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=138, resume processing ppid=137 2024-12-09T03:20:34,457 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=138, ppid=137, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 286 msec 2024-12-09T03:20:34,456 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-12-09T03:20:34,458 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=137, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 427 msec 2024-12-09T03:20:34,492 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2d05c4a2 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3fcd260a 2024-12-09T03:20:34,497 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:34,497 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:34,498 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@7a2585ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:34,499 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:34,501 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:34,502 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:34,512 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f196006e connected 2024-12-09T03:20:34,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=137 2024-12-09T03:20:34,657 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:34,658 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:34,659 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:34,659 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:34,664 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:34,665 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-09T03:20:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=139, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:34,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139 2024-12-09T03:20:34,684 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:34,684 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:34,688 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:34,696 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196006f connected 2024-12-09T03:20:34,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139 2024-12-09T03:20:34,798 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196006f, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:34,802 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:34,954 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:34,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=140 2024-12-09T03:20:34,955 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:34,957 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:34,957 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:34,957 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:34,957 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f196006e, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:34,957 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f196006e, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:34,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139 2024-12-09T03:20:35,060 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f196006e, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:35,061 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=140}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2d05c4a2 to 127.0.0.1:63605 2024-12-09T03:20:35,061 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=140}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:35,125 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:35,126 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:35,126 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-12-09T03:20:35,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=140 2024-12-09T03:20:35,128 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:35,130 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=140, resume processing ppid=139 2024-12-09T03:20:35,130 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=140, ppid=139, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 326 msec 2024-12-09T03:20:35,130 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-12-09T03:20:35,131 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=139, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 465 msec 2024-12-09T03:20:35,171 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2de7c5b6 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@b67a833 2024-12-09T03:20:35,176 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:35,177 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:35,177 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@5c270116, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:35,178 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:35,182 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:35,184 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:35,188 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960071 connected 2024-12-09T03:20:35,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139 2024-12-09T03:20:35,297 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:35,298 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:35,298 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:35,299 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:35,300 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T03:20:35,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=141, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:35,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=141 2024-12-09T03:20:35,324 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:35,325 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:35,328 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:35,329 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960072 connected 2024-12-09T03:20:35,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=141 2024-12-09T03:20:35,431 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960072, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:35,435 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:35,586 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:35,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=142 2024-12-09T03:20:35,587 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:35,588 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:35,589 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:35,589 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:35,589 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960071, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:35,589 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960071, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:35,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=141 2024-12-09T03:20:35,692 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=142}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2de7c5b6 to 127.0.0.1:63605 2024-12-09T03:20:35,692 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960071, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:35,692 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=142}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:35,710 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:35,710 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:35,711 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-12-09T03:20:35,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=142 2024-12-09T03:20:35,713 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:35,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=142, resume processing ppid=141 2024-12-09T03:20:35,715 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=142, ppid=141, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 278 msec 2024-12-09T03:20:35,715 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T03:20:35,716 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=141, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 415 msec 2024-12-09T03:20:35,743 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x433ad846 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1c7c9f62 2024-12-09T03:20:35,747 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:35,748 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:35,748 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@7d2857a9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:35,749 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:35,752 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:35,752 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:35,758 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960074 connected 2024-12-09T03:20:35,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=141 2024-12-09T03:20:35,928 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:35,929 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:35,930 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:35,931 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-09T03:20:35,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=143, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:35,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143 2024-12-09T03:20:35,951 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:35,952 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:35,954 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:35,955 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960075 connected 2024-12-09T03:20:36,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143 2024-12-09T03:20:36,056 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960075, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:36,060 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:36,212 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:36,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=144 2024-12-09T03:20:36,213 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:36,214 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:36,214 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:36,214 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:36,214 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960074, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:36,215 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960074, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:36,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143 2024-12-09T03:20:36,317 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960074, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:36,317 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=144}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x433ad846 to 127.0.0.1:63605 2024-12-09T03:20:36,317 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=144}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:36,336 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:36,336 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:36,337 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-12-09T03:20:36,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=144 2024-12-09T03:20:36,339 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:36,341 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=144, resume processing ppid=143 2024-12-09T03:20:36,341 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-12-09T03:20:36,341 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=144, ppid=143, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 279 msec 2024-12-09T03:20:36,342 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=143, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 410 msec 2024-12-09T03:20:36,369 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7a30741f to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@735ac471 2024-12-09T03:20:36,374 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:36,374 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:36,374 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@5fe3ae31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:36,375 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:36,378 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:36,379 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960077 connected 2024-12-09T03:20:36,379 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:36,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143 2024-12-09T03:20:36,567 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:36,567 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:36,568 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T03:20:36,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=145, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:36,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=145 2024-12-09T03:20:36,591 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:36,592 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:36,594 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:36,595 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960078 connected 2024-12-09T03:20:36,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=145 2024-12-09T03:20:36,696 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960078, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:36,700 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:36,852 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:36,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=146 2024-12-09T03:20:36,853 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:36,854 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:36,854 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:36,854 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:36,855 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960077, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:36,855 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960077, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:36,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=145 2024-12-09T03:20:36,958 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960077, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:36,958 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=146}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7a30741f to 127.0.0.1:63605 2024-12-09T03:20:36,958 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=146}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:36,979 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:36,979 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:36,979 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=146 2024-12-09T03:20:36,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=146 2024-12-09T03:20:36,982 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:36,984 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=146, resume processing ppid=145 2024-12-09T03:20:36,984 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T03:20:36,984 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=146, ppid=145, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 282 msec 2024-12-09T03:20:36,985 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=145, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 416 msec 2024-12-09T03:20:37,022 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0bc3a648 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3b55aff 2024-12-09T03:20:37,027 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:37,027 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:37,027 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@32fe3fd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:37,028 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:37,031 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:37,031 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:37,031 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f196007a connected 2024-12-09T03:20:37,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=145 2024-12-09T03:20:37,197 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:37,197 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:37,198 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:37,198 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:37,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=147, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:37,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=148, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-12-09T03:20:37,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-12-09T03:20:37,201 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:37,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:37,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:37,204 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:37,205 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-12-09T03:20:37,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-12-09T03:20:37,356 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:37,357 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=149 2024-12-09T03:20:37,357 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T03:20:37,357 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T03:20:37,357 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T03:20:37,357 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:37,357 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f196007a, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:37,358 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f196007a, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:37,460 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f196007a, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:37,460 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=149}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0bc3a648 to 127.0.0.1:63605 2024-12-09T03:20:37,460 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=149}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:37,460 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T03:20:37,462 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=149}] zookeeper.ZKUtil(111): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:37,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:37,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:37,470 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:37,471 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=149}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:37,471 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-12-09T03:20:37,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=149 2024-12-09T03:20:37,473 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:37,474 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=149, resume processing ppid=147 2024-12-09T03:20:37,474 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=149, ppid=147, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 268 msec 2024-12-09T03:20:37,477 DEBUG [PEWorker-3 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:37,477 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T03:20:37,478 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=147, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 279 msec 2024-12-09T03:20:37,479 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:37,480 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=148, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:37,481 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=148, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=282 msec 2024-12-09T03:20:37,481 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:37,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-12-09T03:20:37,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-12-09T03:20:37,517 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:37,518 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T03:20:37,518 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:37,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=150, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:37,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=150 2024-12-09T03:20:37,520 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:37,521 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=150, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:37,522 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=150, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T03:20:37,522 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:37,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=150 2024-12-09T03:20:37,627 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:37,641 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[1] Thread=222 (was 222), OpenFileDescriptor=450 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=537 (was 523) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9233 (was 9330) 2024-12-09T03:20:37,650 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[1] Thread=222, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=537, ProcessCount=11, AvailableMemoryMB=9233 2024-12-09T03:20:37,652 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:37,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=151, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:37,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-12-09T03:20:37,680 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:37,681 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:37,684 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:37,685 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196007b connected 2024-12-09T03:20:37,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-12-09T03:20:37,786 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196007b, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:37,794 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=152, ppid=151, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:37,946 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:37,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=152 2024-12-09T03:20:37,947 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T03:20:37,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-12-09T03:20:37,968 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=152}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:37,990 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:37,998 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=152}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=152 2024-12-09T03:20:37,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=152 2024-12-09T03:20:38,001 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:38,002 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=152, resume processing ppid=151 2024-12-09T03:20:38,002 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=152, ppid=151, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 207 msec 2024-12-09T03:20:38,003 INFO [PEWorker-4 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T03:20:38,004 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=151, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 351 msec 2024-12-09T03:20:38,043 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x70c58bec to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@62d2d1f7 2024-12-09T03:20:38,051 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:38,052 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:38,052 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@1ff32b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:38,053 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:38,056 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:38,056 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:38,056 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f196007d connected 2024-12-09T03:20:38,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=151 2024-12-09T03:20:38,277 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:38,278 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T03:20:38,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=153, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:38,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-12-09T03:20:38,302 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:38,302 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:38,306 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:38,332 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196007e connected 2024-12-09T03:20:38,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-12-09T03:20:38,434 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196007e, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:38,437 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=154, ppid=153, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:38,589 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:38,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=154 2024-12-09T03:20:38,590 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:38,591 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:38,591 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:38,591 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:38,591 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f196007d, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:38,592 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f196007d, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:38,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-12-09T03:20:38,694 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f196007d, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:38,694 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=154}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x70c58bec to 127.0.0.1:63605 2024-12-09T03:20:38,694 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=154}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:38,717 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:38,717 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:38,717 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=154}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=154 2024-12-09T03:20:38,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=154 2024-12-09T03:20:38,720 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:38,721 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=154, resume processing ppid=153 2024-12-09T03:20:38,722 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=154, ppid=153, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 283 msec 2024-12-09T03:20:38,722 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T03:20:38,724 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=153, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 444 msec 2024-12-09T03:20:38,752 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x70b2b314 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5fdc6483 2024-12-09T03:20:38,767 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:38,767 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:38,768 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@11678807, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:38,770 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:38,772 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:38,772 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:38,772 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960080 connected 2024-12-09T03:20:38,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=153 2024-12-09T03:20:38,907 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:38,908 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:38,909 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-09T03:20:38,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=155, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:38,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-12-09T03:20:38,933 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:38,934 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:38,936 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:38,937 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960081 connected 2024-12-09T03:20:39,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-12-09T03:20:39,038 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960081, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:39,042 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=156, ppid=155, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:39,194 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:39,194 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=156 2024-12-09T03:20:39,195 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=156}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:39,196 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=156}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:39,196 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=156}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:39,196 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:39,196 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960080, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:39,196 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960080, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:39,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-12-09T03:20:39,298 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960080, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:39,298 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=156}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x70b2b314 to 127.0.0.1:63605 2024-12-09T03:20:39,299 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=156}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:39,321 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=156}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:39,321 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=156}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:39,321 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=156}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=156 2024-12-09T03:20:39,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=156 2024-12-09T03:20:39,324 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:39,325 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=156, resume processing ppid=155 2024-12-09T03:20:39,325 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=156, ppid=155, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 282 msec 2024-12-09T03:20:39,326 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-09T03:20:39,327 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=155, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 417 msec 2024-12-09T03:20:39,357 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4696dfd1 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@126a0f6f 2024-12-09T03:20:39,361 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:39,362 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:39,362 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@5f8b116b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:39,363 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:39,366 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:39,366 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:39,366 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960083 connected 2024-12-09T03:20:39,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=155 2024-12-09T03:20:39,537 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:39,537 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:39,539 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns1:testNamespacesAndTableCfsConfigConflict1=null},bandwidth=0,serial=false 2024-12-09T03:20:39,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=157, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:39,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-12-09T03:20:39,563 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:39,563 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:39,566 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:39,567 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960084 connected 2024-12-09T03:20:39,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-12-09T03:20:39,669 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:39,669 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960084, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:39,671 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=157, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:39,672 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=157, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=132 msec 2024-12-09T03:20:39,672 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:39,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=157 2024-12-09T03:20:39,858 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config 2024-12-09T03:20:39,859 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:39,860 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-09T03:20:39,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=158, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:39,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=158 2024-12-09T03:20:39,885 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:39,885 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:39,896 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:39,897 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960085 connected 2024-12-09T03:20:39,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=158 2024-12-09T03:20:39,998 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960085, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:40,003 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:40,156 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:40,156 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=159 2024-12-09T03:20:40,157 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=159}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:40,158 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=159}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:40,158 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=159}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:40,158 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:40,158 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960083, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:40,159 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960083, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:40,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=158 2024-12-09T03:20:40,261 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960083, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:40,261 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=159}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4696dfd1 to 127.0.0.1:63605 2024-12-09T03:20:40,261 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=159}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:40,282 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=159}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:40,282 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=159}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:40,282 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-12-09T03:20:40,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=159 2024-12-09T03:20:40,285 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:40,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=159, resume processing ppid=158 2024-12-09T03:20:40,287 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=159, ppid=158, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 282 msec 2024-12-09T03:20:40,287 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-09T03:20:40,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=158, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 427 msec 2024-12-09T03:20:40,315 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x716ab3e4 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@f37b9c3 2024-12-09T03:20:40,318 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:40,319 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:40,319 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@5fbf5113, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:40,319 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:40,322 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:40,322 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:40,323 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960087 connected 2024-12-09T03:20:40,338 INFO [master/66b6cc25935e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-12-09T03:20:40,338 INFO [master/66b6cc25935e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-12-09T03:20:40,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=158 2024-12-09T03:20:40,487 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:40,487 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:40,488 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-12-09T03:20:40,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=160, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:40,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=160 2024-12-09T03:20:40,509 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:40,510 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:40,516 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:40,517 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960088 connected 2024-12-09T03:20:40,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=160 2024-12-09T03:20:40,619 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960088, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:40,619 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:40,620 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=160, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:40,621 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=160, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=131 msec 2024-12-09T03:20:40,621 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:40,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=160 2024-12-09T03:20:40,808 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config 2024-12-09T03:20:40,808 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:40,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=161, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:40,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=161 2024-12-09T03:20:40,811 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:40,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:40,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:40,814 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:40,814 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=162, ppid=161, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:40,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=161 2024-12-09T03:20:40,965 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:40,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=162 2024-12-09T03:20:40,966 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=162}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T03:20:40,966 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=162}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T03:20:40,966 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=162}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T03:20:40,966 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:40,966 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960087, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:40,966 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960087, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:41,069 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960087, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:41,069 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=162}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x716ab3e4 to 127.0.0.1:63605 2024-12-09T03:20:41,069 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=162}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:41,069 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=162}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T03:20:41,076 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=162}] zookeeper.ZKUtil(111): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:41,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:41,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:41,078 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:41,079 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=162}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:41,079 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=162}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=162 2024-12-09T03:20:41,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=162 2024-12-09T03:20:41,081 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:41,082 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=162, resume processing ppid=161 2024-12-09T03:20:41,082 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=162, ppid=161, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 267 msec 2024-12-09T03:20:41,085 DEBUG [PEWorker-2 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:41,085 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T03:20:41,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=161, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 277 msec 2024-12-09T03:20:41,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=161 2024-12-09T03:20:41,127 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:41,128 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:41,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=163, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:41,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=163 2024-12-09T03:20:41,130 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:41,131 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=163, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:41,131 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=163, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T03:20:41,132 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:41,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=163 2024-12-09T03:20:41,237 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T03:20:41,237 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:41,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=164, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:41,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=164 2024-12-09T03:20:41,239 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:41,240 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=164, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:41,241 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=164, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T03:20:41,241 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:41,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=164 2024-12-09T03:20:41,347 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:41,364 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[1] Thread=220 (was 222), OpenFileDescriptor=448 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=550 (was 537) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9222 (was 9233) 2024-12-09T03:20:41,373 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAddRemovePeer[1] Thread=220, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=550, ProcessCount=11, AvailableMemoryMB=9221 2024-12-09T03:20:41,374 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=165, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:41,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-12-09T03:20:41,398 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:41,399 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:41,401 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:41,402 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960089 connected 2024-12-09T03:20:41,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-12-09T03:20:41,504 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960089, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:41,510 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:41,661 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:41,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=166 2024-12-09T03:20:41,662 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=166}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T03:20:41,679 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=166}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:41,695 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=166}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:41,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-12-09T03:20:41,705 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=166}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=166 2024-12-09T03:20:41,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=166 2024-12-09T03:20:41,707 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:41,709 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=166, resume processing ppid=165 2024-12-09T03:20:41,709 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=166, ppid=165, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 197 msec 2024-12-09T03:20:41,710 INFO [PEWorker-2 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T03:20:41,711 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=165, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 336 msec 2024-12-09T03:20:41,745 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2496473f to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@52e20537 2024-12-09T03:20:41,749 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:41,749 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:41,749 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@332e8c03, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:41,750 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:41,758 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:41,758 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:41,759 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f196008b connected 2024-12-09T03:20:42,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=165 2024-12-09T03:20:42,007 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:42,008 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=167, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:42,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=167 2024-12-09T03:20:42,031 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:42,031 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:42,035 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:42,036 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196008c connected 2024-12-09T03:20:42,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=167 2024-12-09T03:20:42,137 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196008c, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:42,138 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:121) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:42,139 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=167, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:121) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:42,140 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=167, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=131 msec 2024-12-09T03:20:42,140 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:42,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=167 2024-12-09T03:20:42,327 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Replication peer 1 already exists 2024-12-09T03:20:42,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-09T03:20:42,328 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:42,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=168, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:42,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=168 2024-12-09T03:20:42,330 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:42,330 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=168, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:42,331 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=168, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T03:20:42,331 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:42,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=168 2024-12-09T03:20:42,437 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:42,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-09T03:20:42,439 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=2, config=clusterKey=127.0.0.1:63605:/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:42,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=169, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:42,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=169 2024-12-09T03:20:42,456 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:42,456 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:42,459 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:42,460 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196008d connected 2024-12-09T03:20:42,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=169 2024-12-09T03:20:42,564 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196008d, quorum=127.0.0.1:63605, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:42,570 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:42,721 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:42,722 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=170 2024-12-09T03:20:42,722 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=2, type=ADD_PEER 2024-12-09T03:20:42,744 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=170}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:42,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=169 2024-12-09T03:20:42,759 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 2000 millis 2024-12-09T03:20:42,766 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.ReplicationSource(229): queueId=2, ReplicationSource: 2, currentBandwidth=0 2024-12-09T03:20:42,771 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=170}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=170 2024-12-09T03:20:42,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=170 2024-12-09T03:20:42,773 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 2 for ADD on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:42,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=170, resume processing ppid=169 2024-12-09T03:20:42,777 INFO [PEWorker-4 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 2, config clusterKey=127.0.0.1:63605:/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T03:20:42,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=170, ppid=169, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 204 msec 2024-12-09T03:20:42,779 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=169, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 338 msec 2024-12-09T03:20:42,815 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7996d006 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@28f86ad8 2024-12-09T03:20:42,820 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,2 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:42,820 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,2 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:42,821 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@400256f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:42,822 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 2 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:42,824 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,2-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 20x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:42,824 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,2 {}] regionserver.ReplicationSource(569): peerId=2, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:42,828 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 2-0x10074c9f196008f connected 2024-12-09T03:20:42,962 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T03:20:43,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=169 2024-12-09T03:20:43,067 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 2 completed 2024-12-09T03:20:43,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-09T03:20:43,068 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:43,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=171, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:43,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=171 2024-12-09T03:20:43,072 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:43,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:43,074 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:43,075 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:43,075 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=172, ppid=171, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:43,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=171 2024-12-09T03:20:43,227 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:43,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=172 2024-12-09T03:20:43,228 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T03:20:43,228 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T03:20:43,229 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T03:20:43,229 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:43,229 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f196008b, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:43,229 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f196008b, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:43,332 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f196008b, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:43,332 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=172}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2496473f to 127.0.0.1:63605 2024-12-09T03:20:43,332 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=172}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:43,332 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T03:20:43,334 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=172}] zookeeper.ZKUtil(111): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:43,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:43,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:43,337 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:43,337 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=172}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:43,337 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=172}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=172 2024-12-09T03:20:43,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=172 2024-12-09T03:20:43,340 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:43,341 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=172, resume processing ppid=171 2024-12-09T03:20:43,342 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=172, ppid=171, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 265 msec 2024-12-09T03:20:43,346 DEBUG [PEWorker-2 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:43,346 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T03:20:43,348 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=171, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 278 msec 2024-12-09T03:20:43,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=171 2024-12-09T03:20:43,387 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:43,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-09T03:20:43,388 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:43,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=173, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:43,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=173 2024-12-09T03:20:43,400 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/peer-state 2024-12-09T03:20:43,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/peer-state 2024-12-09T03:20:43,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/2 2024-12-09T03:20:43,403 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2 2024-12-09T03:20:43,404 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=174, ppid=173, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=173 2024-12-09T03:20:43,556 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:43,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=174 2024-12-09T03:20:43,557 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=174}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=2, type=REMOVE_PEER 2024-12-09T03:20:43,557 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=174}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 2: 0 2024-12-09T03:20:43,557 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=174}] regionserver.ReplicationSource(657): peerId=2, Closing source 2 because: Replication stream was removed by a user 2024-12-09T03:20:43,557 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,2 {}] regionserver.ReplicationSource(508): peerId=2, Interrupted while sleeping between retries 2024-12-09T03:20:43,557 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,2 {}] zookeeper.ZKUtil(165): connection to cluster: 2-0x10074c9f196008f, quorum=127.0.0.1:63605, baseZNode=/hbase-test2 Unable to set watcher on znode (/hbase-test2/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:43,557 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,2 {}] zookeeper.ZKWatcher(734): connection to cluster: 2-0x10074c9f196008f, quorum=127.0.0.1:63605, baseZNode=/hbase-test2 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:43,663 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=174}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7996d006 to 127.0.0.1:63605 2024-12-09T03:20:43,663 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=174}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:43,664 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=174}] regionserver.ReplicationSourceManager(430): Done with the queue 2 2024-12-09T03:20:43,664 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,2-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 2-0x10074c9f196008f, quorum=127.0.0.1:63605, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:43,666 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=174}] zookeeper.ZKUtil(111): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/2/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:43,667 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/2/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:43,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/2 2024-12-09T03:20:43,668 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/2 2024-12-09T03:20:43,668 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=174}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/2 not found in hfile reference queue. 2024-12-09T03:20:43,668 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=174}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=174 2024-12-09T03:20:43,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=174 2024-12-09T03:20:43,670 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 2 for REMOVE on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:43,672 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=174, resume processing ppid=173 2024-12-09T03:20:43,672 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=174, ppid=173, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 266 msec 2024-12-09T03:20:43,675 DEBUG [PEWorker-3 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/2 not found in hfile reference queue. 2024-12-09T03:20:43,676 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(76): Successfully removed peer 2 2024-12-09T03:20:43,677 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=173, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 288 msec 2024-12-09T03:20:43,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=173 2024-12-09T03:20:43,708 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 2 completed 2024-12-09T03:20:43,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-09T03:20:43,710 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:43,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=175, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:43,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=175 2024-12-09T03:20:43,712 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:43,713 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=175, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:43,715 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=175, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T03:20:43,715 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:43,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=175 2024-12-09T03:20:43,817 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T03:20:43,817 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:43,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=176, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=176 2024-12-09T03:20:43,819 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:43,820 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=176, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:43,821 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=176, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T03:20:43,821 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:43,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=176 2024-12-09T03:20:43,927 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:43,941 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAddRemovePeer[1] Thread=220 (was 220), OpenFileDescriptor=446 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=562 (was 550) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9207 (was 9221) 2024-12-09T03:20:43,953 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[1] Thread=220, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=562, ProcessCount=11, AvailableMemoryMB=9207 2024-12-09T03:20:43,954 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=whatever,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=177, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:43,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=177 2024-12-09T03:20:43,956 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:291) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:288) ~[classes/:?] ... 12 more 2024-12-09T03:20:43,957 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=177, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:291) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:288) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:43,958 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=177, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=3 msec 2024-12-09T03:20:43,958 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:44,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=177 2024-12-09T03:20:44,067 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Can not instantiate configured replication endpoint class=whatever 2024-12-09T03:20:44,067 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:44,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=178, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:44,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=178 2024-12-09T03:20:44,069 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:44,070 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=178, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:44,071 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=178, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T03:20:44,071 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:44,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=178 2024-12-09T03:20:44,181 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T03:20:44,183 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:44,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=179, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:44,186 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:44,186 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=179, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:44,187 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=179, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-09T03:20:44,187 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:44,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=179 2024-12-09T03:20:44,190 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:44,204 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[1] Thread=220 (was 220), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=562 (was 562), ProcessCount=11 (was 11), AvailableMemoryMB=9207 (was 9207) 2024-12-09T03:20:44,214 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[1] Thread=220, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=562, ProcessCount=11, AvailableMemoryMB=9207 2024-12-09T03:20:44,215 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=whatever,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=180, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:44,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=180 2024-12-09T03:20:44,217 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] ... 13 more 2024-12-09T03:20:44,218 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=180, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:44,219 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=180, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=3 msec 2024-12-09T03:20:44,219 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:44,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=180 2024-12-09T03:20:44,327 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Invalid cluster key: whatever 2024-12-09T03:20:44,327 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:44,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=181, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:44,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=181 2024-12-09T03:20:44,329 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:44,330 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=181, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:44,331 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=181, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-09T03:20:44,331 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:44,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=181 2024-12-09T03:20:44,437 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T03:20:44,437 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:44,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=182, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:44,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=182 2024-12-09T03:20:44,439 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:44,440 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=182, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:44,441 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=182, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T03:20:44,441 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:44,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=182 2024-12-09T03:20:44,547 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:44,560 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[1] Thread=220 (was 220), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=562 (was 562), ProcessCount=11 (was 11), AvailableMemoryMB=9206 (was 9207) 2024-12-09T03:20:44,570 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[1] Thread=220, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=562, ProcessCount=11, AvailableMemoryMB=9206 2024-12-09T03:20:44,571 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:44,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=183, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:44,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=183 2024-12-09T03:20:44,588 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:44,589 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:44,591 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:44,591 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960090 connected 2024-12-09T03:20:44,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=183 2024-12-09T03:20:44,693 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960090, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:44,699 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=184, ppid=183, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:44,851 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:44,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=184 2024-12-09T03:20:44,852 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T03:20:44,875 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=184}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:44,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=183 2024-12-09T03:20:44,895 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:44,905 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=184}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=184 2024-12-09T03:20:44,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=184 2024-12-09T03:20:44,908 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:44,910 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=184, resume processing ppid=183 2024-12-09T03:20:44,910 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=184, ppid=183, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 209 msec 2024-12-09T03:20:44,910 INFO [PEWorker-2 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T03:20:44,911 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=183, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 339 msec 2024-12-09T03:20:44,937 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x05f3545d to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7534defd 2024-12-09T03:20:44,941 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:44,941 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:44,942 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@4aa2768c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:44,942 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:44,949 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:44,950 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:44,950 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960092 connected 2024-12-09T03:20:45,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=183 2024-12-09T03:20:45,197 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:45,197 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T03:20:45,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=185, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:45,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=185 2024-12-09T03:20:45,221 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:45,221 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:45,224 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:45,224 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960093 connected 2024-12-09T03:20:45,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=185 2024-12-09T03:20:45,327 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960093, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:45,330 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=186, ppid=185, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:45,482 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:45,483 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=186 2024-12-09T03:20:45,483 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:45,484 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:45,485 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:45,485 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:45,485 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960092, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:45,485 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960092, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:45,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=185 2024-12-09T03:20:45,590 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960092, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:45,590 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=186}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x05f3545d to 127.0.0.1:63605 2024-12-09T03:20:45,590 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=186}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:45,620 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:45,620 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:45,621 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=186}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=186 2024-12-09T03:20:45,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=186 2024-12-09T03:20:45,623 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:45,625 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=186, resume processing ppid=185 2024-12-09T03:20:45,625 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=186, ppid=185, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 293 msec 2024-12-09T03:20:45,625 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T03:20:45,626 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=185, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 428 msec 2024-12-09T03:20:45,677 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x01890159 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1505b318 2024-12-09T03:20:45,683 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:45,683 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:45,684 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@287256a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:45,685 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:45,689 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:45,692 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:45,695 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960095 connected 2024-12-09T03:20:45,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=185 2024-12-09T03:20:45,827 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:45,828 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:45,828 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-12-09T03:20:45,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=187, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:45,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=187 2024-12-09T03:20:45,852 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:45,853 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:45,865 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:45,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=187 2024-12-09T03:20:45,967 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:45,971 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=188, ppid=187, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:46,124 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:46,124 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=188 2024-12-09T03:20:46,124 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:46,126 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:46,126 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:46,126 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:46,126 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960095, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:46,126 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960095, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=187 2024-12-09T03:20:46,228 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960095, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:46,228 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=188}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x01890159 to 127.0.0.1:63605 2024-12-09T03:20:46,228 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=188}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:46,244 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:46,245 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:46,245 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=188}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=188 2024-12-09T03:20:46,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=188 2024-12-09T03:20:46,247 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:46,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=188, resume processing ppid=187 2024-12-09T03:20:46,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=188, ppid=187, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 276 msec 2024-12-09T03:20:46,248 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-12-09T03:20:46,249 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=187, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 420 msec 2024-12-09T03:20:46,272 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2f702ad6 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@d8aee60 2024-12-09T03:20:46,275 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:46,275 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:46,275 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@7453191, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:46,276 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:46,280 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:46,280 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:46,280 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f1960098 connected 2024-12-09T03:20:46,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=187 2024-12-09T03:20:46,457 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:46,457 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:46,458 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:46,458 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-09T03:20:46,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=189, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:46,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=189 2024-12-09T03:20:46,474 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:46,474 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:46,476 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:46,477 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960099 connected 2024-12-09T03:20:46,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=189 2024-12-09T03:20:46,578 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f1960099, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:46,581 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=190, ppid=189, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:46,732 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:46,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=190 2024-12-09T03:20:46,733 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:46,734 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:46,734 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:46,734 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:46,734 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f1960098, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:46,734 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f1960098, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:46,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=189 2024-12-09T03:20:46,836 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f1960098, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:46,836 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=190}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2f702ad6 to 127.0.0.1:63605 2024-12-09T03:20:46,836 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=190}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:46,850 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:46,851 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:46,851 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=190}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=190 2024-12-09T03:20:46,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=190 2024-12-09T03:20:46,852 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:46,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=190, resume processing ppid=189 2024-12-09T03:20:46,854 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=190, ppid=189, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 271 msec 2024-12-09T03:20:46,854 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-12-09T03:20:46,854 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=189, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 396 msec 2024-12-09T03:20:46,878 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7ec7bd23 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@237baf09 2024-12-09T03:20:46,880 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:46,881 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:46,881 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@61789dd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:46,882 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:46,884 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:46,884 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:46,884 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f196009b connected 2024-12-09T03:20:47,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=189 2024-12-09T03:20:47,087 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:47,087 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:47,088 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:47,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=191, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:47,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=191 2024-12-09T03:20:47,090 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:47,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:47,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:47,092 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:47,093 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=192, ppid=191, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:47,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=191 2024-12-09T03:20:47,244 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:47,245 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=192 2024-12-09T03:20:47,245 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=192}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T03:20:47,245 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=192}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T03:20:47,245 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=192}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T03:20:47,245 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:47,246 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f196009b, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:47,246 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f196009b, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:47,347 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f196009b, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:47,348 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=192}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7ec7bd23 to 127.0.0.1:63605 2024-12-09T03:20:47,348 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=192}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:47,348 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=192}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T03:20:47,350 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=192}] zookeeper.ZKUtil(111): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:47,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:47,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:47,352 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:47,352 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=192}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:47,352 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=192}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=192 2024-12-09T03:20:47,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=192 2024-12-09T03:20:47,354 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:47,355 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=192, resume processing ppid=191 2024-12-09T03:20:47,355 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=192, ppid=191, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 261 msec 2024-12-09T03:20:47,357 DEBUG [PEWorker-4 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:47,357 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T03:20:47,358 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=191, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 269 msec 2024-12-09T03:20:47,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=191 2024-12-09T03:20:47,407 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:47,407 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:47,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=193, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:47,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=193 2024-12-09T03:20:47,409 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:47,410 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=193, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:47,410 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=193, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T03:20:47,410 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:47,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=193 2024-12-09T03:20:47,517 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T03:20:47,518 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:47,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=194, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:47,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=194 2024-12-09T03:20:47,519 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:47,520 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=194, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:47,521 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=194, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-09T03:20:47,521 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:47,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=194 2024-12-09T03:20:47,627 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:47,638 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[1] Thread=220 (was 220), OpenFileDescriptor=446 (was 446), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=562 (was 562), ProcessCount=11 (was 11), AvailableMemoryMB=9185 (was 9206) 2024-12-09T03:20:47,646 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[1] Thread=220, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=562, ProcessCount=11, AvailableMemoryMB=9185 2024-12-09T03:20:47,647 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=195, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=195 2024-12-09T03:20:47,662 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:47,663 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:47,665 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:47,666 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196009c connected 2024-12-09T03:20:47,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=195 2024-12-09T03:20:47,767 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196009c, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:47,772 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=196, ppid=195, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:47,923 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:47,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=196 2024-12-09T03:20:47,924 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T03:20:47,939 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=196}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:47,952 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:47,959 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=196}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=196 2024-12-09T03:20:47,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=196 2024-12-09T03:20:47,961 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:47,962 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=196, resume processing ppid=195 2024-12-09T03:20:47,962 INFO [PEWorker-5 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T03:20:47,962 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=196, ppid=195, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 189 msec 2024-12-09T03:20:47,963 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=195, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 316 msec 2024-12-09T03:20:47,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=195 2024-12-09T03:20:47,967 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:47,967 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T03:20:47,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=197, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:47,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=197 2024-12-09T03:20:47,988 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:47,989 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:47,991 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:47,992 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196009d connected 2024-12-09T03:20:47,994 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x05a71435 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7ce8c09a 2024-12-09T03:20:47,996 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:47,997 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:47,997 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@63b6c6a8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:47,997 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:48,000 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:48,000 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:48,000 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f196009f connected 2024-12-09T03:20:48,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=197 2024-12-09T03:20:48,093 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f196009d, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:48,096 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=198, ppid=197, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:48,247 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:48,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=198 2024-12-09T03:20:48,248 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:48,249 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:48,249 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:48,249 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:48,249 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f196009f, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:48,249 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f196009f, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=197 2024-12-09T03:20:48,351 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f196009f, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:48,352 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=198}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x05a71435 to 127.0.0.1:63605 2024-12-09T03:20:48,352 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=198}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:48,366 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:48,367 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:48,367 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=198}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=198 2024-12-09T03:20:48,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=198 2024-12-09T03:20:48,369 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:48,370 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=198, resume processing ppid=197 2024-12-09T03:20:48,370 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-12-09T03:20:48,370 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=198, ppid=197, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 273 msec 2024-12-09T03:20:48,371 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=197, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 403 msec 2024-12-09T03:20:48,394 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x64c7ac4e to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@159bdf12 2024-12-09T03:20:48,397 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:48,397 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:48,398 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@4914cf06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:48,398 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:48,403 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:48,403 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:48,403 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f19600a1 connected 2024-12-09T03:20:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=197 2024-12-09T03:20:48,596 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:48,597 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:48,597 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-12-09T03:20:48,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=199, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=199 2024-12-09T03:20:48,614 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:48,615 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:48,617 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:48,617 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f19600a2 connected 2024-12-09T03:20:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=199 2024-12-09T03:20:48,720 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f19600a2, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:48,723 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=200, ppid=199, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:48,834 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T03:20:48,874 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:48,875 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=200 2024-12-09T03:20:48,875 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:48,876 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:48,876 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:48,876 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:48,876 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f19600a1, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:48,877 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f19600a1, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:48,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=199 2024-12-09T03:20:48,978 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f19600a1, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:48,979 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=200}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x64c7ac4e to 127.0.0.1:63605 2024-12-09T03:20:48,979 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=200}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:48,993 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:48,993 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:48,993 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=200}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=200 2024-12-09T03:20:48,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=200 2024-12-09T03:20:48,995 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:48,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=200, resume processing ppid=199 2024-12-09T03:20:48,996 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=200, ppid=199, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 272 msec 2024-12-09T03:20:48,996 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-12-09T03:20:48,997 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=199, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 399 msec 2024-12-09T03:20:49,021 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x688abe1e to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@d82680b 2024-12-09T03:20:49,024 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:49,024 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:49,025 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@2cd8267a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:49,025 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:49,027 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:49,028 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:49,028 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f19600a4 connected 2024-12-09T03:20:49,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=199 2024-12-09T03:20:49,227 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:49,227 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:49,228 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:49,228 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-09T03:20:49,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=201, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:49,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=201 2024-12-09T03:20:49,244 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:49,244 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:49,247 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:49,247 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f19600a5 connected 2024-12-09T03:20:49,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=201 2024-12-09T03:20:49,349 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f19600a5, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:49,352 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=202, ppid=201, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:49,503 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:49,504 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=202 2024-12-09T03:20:49,504 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:49,505 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:49,505 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:49,505 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:49,505 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f19600a4, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:49,506 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f19600a4, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=201 2024-12-09T03:20:49,608 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f19600a4, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:49,608 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=202}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x688abe1e to 127.0.0.1:63605 2024-12-09T03:20:49,608 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=202}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:49,622 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:49,622 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:49,622 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=202}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=202 2024-12-09T03:20:49,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=202 2024-12-09T03:20:49,624 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:49,626 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=202, resume processing ppid=201 2024-12-09T03:20:49,626 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-09T03:20:49,626 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=202, ppid=201, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 272 msec 2024-12-09T03:20:49,627 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=201, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 398 msec 2024-12-09T03:20:49,650 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2e0c979e to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@281c9f5e 2024-12-09T03:20:49,653 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:49,654 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:49,654 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@2d741f3e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:49,655 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:49,657 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:49,657 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:49,657 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f19600a7 connected 2024-12-09T03:20:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=201 2024-12-09T03:20:49,857 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:49,857 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:49,858 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:49,859 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-09T03:20:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=203, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=203 2024-12-09T03:20:49,875 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:49,875 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:49,878 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:49,878 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f19600a8 connected 2024-12-09T03:20:49,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=203 2024-12-09T03:20:49,980 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f19600a8, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:49,983 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=204, ppid=203, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:50,134 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:50,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=204 2024-12-09T03:20:50,135 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:50,136 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:50,136 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:50,136 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:50,136 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f19600a7, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:50,137 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f19600a7, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:50,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=203 2024-12-09T03:20:50,238 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f19600a7, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:50,239 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=204}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2e0c979e to 127.0.0.1:63605 2024-12-09T03:20:50,239 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=204}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:50,253 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:50,253 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:50,254 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=204}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=204 2024-12-09T03:20:50,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=204 2024-12-09T03:20:50,255 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:50,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=204, resume processing ppid=203 2024-12-09T03:20:50,257 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=204, ppid=203, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 272 msec 2024-12-09T03:20:50,257 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-12-09T03:20:50,258 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=203, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 398 msec 2024-12-09T03:20:50,281 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2a53d388 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@c571b2b 2024-12-09T03:20:50,284 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:50,285 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:50,285 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3f1b608b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:50,286 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:50,288 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:50,288 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:50,288 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f19600aa connected 2024-12-09T03:20:50,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=203 2024-12-09T03:20:50,487 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:50,487 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:50,488 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:50,488 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-09T03:20:50,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=205, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:50,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=205 2024-12-09T03:20:50,504 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:50,504 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:50,506 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:50,507 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f19600ab connected 2024-12-09T03:20:50,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=205 2024-12-09T03:20:50,608 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f19600ab, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:50,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=206, ppid=205, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:50,763 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:50,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=206 2024-12-09T03:20:50,763 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:50,764 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:50,764 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:50,765 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:50,765 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f19600aa, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:50,765 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f19600aa, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:50,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=205 2024-12-09T03:20:50,867 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f19600aa, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:50,867 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=206}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2a53d388 to 127.0.0.1:63605 2024-12-09T03:20:50,867 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=206}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:50,881 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:50,882 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:50,882 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=206}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=206 2024-12-09T03:20:50,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=206 2024-12-09T03:20:50,884 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:50,885 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=206, resume processing ppid=205 2024-12-09T03:20:50,885 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=206, ppid=205, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 273 msec 2024-12-09T03:20:50,885 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-09T03:20:50,886 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=205, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 397 msec 2024-12-09T03:20:50,909 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x50ad1066 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5cbc027f 2024-12-09T03:20:50,912 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:50,912 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:50,913 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@52c93a44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:50,913 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:50,915 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:50,916 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:50,916 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f19600ad connected 2024-12-09T03:20:51,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=205 2024-12-09T03:20:51,117 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:51,117 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:51,118 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:51,118 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-09T03:20:51,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=207, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:51,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=207 2024-12-09T03:20:51,134 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:51,134 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:51,137 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:51,137 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f19600ae connected 2024-12-09T03:20:51,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=207 2024-12-09T03:20:51,238 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f19600ae, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:51,242 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=208, ppid=207, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:51,393 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:51,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=208 2024-12-09T03:20:51,394 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:51,395 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:51,395 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:51,395 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:51,395 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f19600ad, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:51,395 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f19600ad, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:51,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=207 2024-12-09T03:20:51,498 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f19600ad, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:51,498 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=208}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x50ad1066 to 127.0.0.1:63605 2024-12-09T03:20:51,498 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=208}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:51,512 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:51,513 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:51,513 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=208}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=208 2024-12-09T03:20:51,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=208 2024-12-09T03:20:51,515 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:51,516 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=208, resume processing ppid=207 2024-12-09T03:20:51,517 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=208, ppid=207, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 273 msec 2024-12-09T03:20:51,516 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-09T03:20:51,517 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=207, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 399 msec 2024-12-09T03:20:51,551 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x61c47516 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5343d8c0 2024-12-09T03:20:51,554 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:51,555 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:51,555 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@49caa1bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:51,556 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:51,558 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:51,558 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:51,558 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f19600b0 connected 2024-12-09T03:20:51,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=207 2024-12-09T03:20:51,747 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:51,748 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:51,748 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-09T03:20:51,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=209, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:51,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=209 2024-12-09T03:20:51,764 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:51,764 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:51,767 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:51,767 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f19600b1 connected 2024-12-09T03:20:51,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=209 2024-12-09T03:20:51,868 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f19600b1, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:51,872 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=210, ppid=209, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:52,023 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:52,024 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=210 2024-12-09T03:20:52,024 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:52,025 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=210}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=210 2024-12-09T03:20:52,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=210 2024-12-09T03:20:52,027 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:52,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=210, resume processing ppid=209 2024-12-09T03:20:52,028 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=210, ppid=209, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 155 msec 2024-12-09T03:20:52,028 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-12-09T03:20:52,029 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=209, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 280 msec 2024-12-09T03:20:52,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=209 2024-12-09T03:20:52,066 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:52,067 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:52,067 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:52,068 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-12-09T03:20:52,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=211, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:52,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=211 2024-12-09T03:20:52,084 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:52,084 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:52,087 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:52,088 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f19600b2 connected 2024-12-09T03:20:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=211 2024-12-09T03:20:52,189 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f19600b2, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:52,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=212, ppid=211, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:52,345 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:52,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=212 2024-12-09T03:20:52,345 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:52,346 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:52,346 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:52,347 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:52,347 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f19600b0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:52,347 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f19600b0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=211 2024-12-09T03:20:52,449 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f19600b0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:52,449 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=212}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x61c47516 to 127.0.0.1:63605 2024-12-09T03:20:52,449 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=212}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:52,463 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:52,463 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:52,463 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=212}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=212 2024-12-09T03:20:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=212 2024-12-09T03:20:52,465 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:52,467 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=212, resume processing ppid=211 2024-12-09T03:20:52,467 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=212, ppid=211, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 272 msec 2024-12-09T03:20:52,467 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-12-09T03:20:52,468 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=211, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 399 msec 2024-12-09T03:20:52,491 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x21082194 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2b6985b 2024-12-09T03:20:52,494 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:52,494 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:52,495 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@1e6ea3a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:52,495 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:52,497 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:52,498 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:52,498 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f19600b4 connected 2024-12-09T03:20:52,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=211 2024-12-09T03:20:52,696 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:52,697 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:52,697 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-12-09T03:20:52,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=213, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:52,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=213 2024-12-09T03:20:52,721 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:52,722 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:52,724 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:52,724 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f19600b5 connected 2024-12-09T03:20:52,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=213 2024-12-09T03:20:52,826 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f19600b5, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:52,828 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=214, ppid=213, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:52,980 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:52,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=214 2024-12-09T03:20:52,980 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:52,981 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:52,981 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:52,982 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:52,982 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f19600b4, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:52,982 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f19600b4, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:53,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=213 2024-12-09T03:20:53,084 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f19600b4, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:53,084 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=214}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x21082194 to 127.0.0.1:63605 2024-12-09T03:20:53,084 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=214}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:53,098 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:53,098 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:53,098 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=214}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=214 2024-12-09T03:20:53,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=214 2024-12-09T03:20:53,100 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:53,102 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=214, resume processing ppid=213 2024-12-09T03:20:53,102 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=214, ppid=213, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 272 msec 2024-12-09T03:20:53,102 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-12-09T03:20:53,103 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=213, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 405 msec 2024-12-09T03:20:53,126 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6403b520 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4fa5279e 2024-12-09T03:20:53,129 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:53,129 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:53,130 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@34f759d3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:53,130 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:53,133 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:53,133 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:53,133 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f19600b7 connected 2024-12-09T03:20:53,195 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-12-09T03:20:53,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=213 2024-12-09T03:20:53,326 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:53,327 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:53,327 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:53,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=215, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:53,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=215 2024-12-09T03:20:53,330 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:53,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:53,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:53,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:53,332 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=216, ppid=215, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:53,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=215 2024-12-09T03:20:53,484 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:53,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=216 2024-12-09T03:20:53,485 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=216}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T03:20:53,485 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=216}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T03:20:53,485 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=216}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T03:20:53,485 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:53,485 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f19600b7, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:53,485 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f19600b7, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:53,587 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f19600b7, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:53,587 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=216}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6403b520 to 127.0.0.1:63605 2024-12-09T03:20:53,587 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=216}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:53,587 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=216}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T03:20:53,589 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=216}] zookeeper.ZKUtil(111): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:53,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:53,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:53,591 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:53,591 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=216}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:53,591 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=216}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=216 2024-12-09T03:20:53,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=216 2024-12-09T03:20:53,593 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:53,594 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=216, resume processing ppid=215 2024-12-09T03:20:53,594 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=216, ppid=215, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 261 msec 2024-12-09T03:20:53,596 DEBUG [PEWorker-5 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:53,596 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T03:20:53,597 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=215, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 269 msec 2024-12-09T03:20:53,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=215 2024-12-09T03:20:53,647 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:53,647 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:53,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=217, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:53,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=217 2024-12-09T03:20:53,648 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:53,649 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=217, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:53,650 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=217, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-09T03:20:53,650 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:53,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=217 2024-12-09T03:20:53,757 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T03:20:53,757 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:53,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=218, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:53,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=218 2024-12-09T03:20:53,758 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:53,759 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=218, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:53,760 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=218, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-09T03:20:53,760 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:53,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=218 2024-12-09T03:20:53,867 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:53,878 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[1] Thread=217 (was 220), OpenFileDescriptor=448 (was 446) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=491 (was 562), ProcessCount=11 (was 11), AvailableMemoryMB=9188 (was 9185) - AvailableMemoryMB LEAK? - 2024-12-09T03:20:53,886 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[1] Thread=217, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=491, ProcessCount=11, AvailableMemoryMB=9187 2024-12-09T03:20:53,887 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:53,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=219, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:53,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=219 2024-12-09T03:20:53,902 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:53,903 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:53,905 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:53,905 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f19600b8 connected 2024-12-09T03:20:53,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=219 2024-12-09T03:20:54,007 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f19600b8, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:54,012 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=220, ppid=219, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:54,163 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:54,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=220 2024-12-09T03:20:54,164 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T03:20:54,179 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=220}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:54,192 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:54,198 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=220}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=220 2024-12-09T03:20:54,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=220 2024-12-09T03:20:54,200 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:54,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=220, resume processing ppid=219 2024-12-09T03:20:54,201 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=220, ppid=219, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 188 msec 2024-12-09T03:20:54,201 INFO [PEWorker-3 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T03:20:54,202 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=219, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 314 msec 2024-12-09T03:20:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=219 2024-12-09T03:20:54,207 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:54,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-09T03:20:54,208 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3938): Client=jenkins//172.17.0.2 disable replication peer, id=1 2024-12-09T03:20:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=221, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure 2024-12-09T03:20:54,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=221 2024-12-09T03:20:54,212 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=222, ppid=221, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:54,226 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4b11d094 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@be8e032 2024-12-09T03:20:54,228 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:54,229 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:54,229 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@6aaf3e04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:54,230 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:54,232 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:54,232 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:54,236 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f19600ba connected 2024-12-09T03:20:54,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=221 2024-12-09T03:20:54,363 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:54,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=222 2024-12-09T03:20:54,364 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=DISABLE_PEER 2024-12-09T03:20:54,364 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-12-09T03:20:54,364 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-12-09T03:20:54,365 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:54,365 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f19600ba, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:54,365 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f19600ba, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:54,467 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f19600ba, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:54,467 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=222}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4b11d094 to 127.0.0.1:63605 2024-12-09T03:20:54,467 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=222}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:54,482 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:54,482 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-12-09T03:20:54,482 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=222}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=222 2024-12-09T03:20:54,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=222 2024-12-09T03:20:54,484 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for DISABLE on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:54,486 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=222, resume processing ppid=221 2024-12-09T03:20:54,486 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=222, ppid=221, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 272 msec 2024-12-09T03:20:54,486 INFO [PEWorker-5 {}] replication.DisablePeerProcedure(67): Successfully disabled peer 1 2024-12-09T03:20:54,487 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=221, state=SUCCESS; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure in 278 msec 2024-12-09T03:20:54,510 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6ec87dd0 to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@77a93e17 2024-12-09T03:20:54,513 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:54,514 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:54,514 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3a69ae93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:54,514 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:54,517 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:54,517 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:54,517 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f19600bc connected 2024-12-09T03:20:54,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=221 2024-12-09T03:20:54,526 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: DISABLE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:54,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-09T03:20:54,527 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=223, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:54,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=223 2024-12-09T03:20:54,530 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:54,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:54,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:54,532 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:54,532 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=224, ppid=223, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:54,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=223 2024-12-09T03:20:54,684 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:54,684 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=224 2024-12-09T03:20:54,684 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=224}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T03:20:54,685 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=224}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T03:20:54,685 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=224}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T03:20:54,685 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:54,685 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f19600bc, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:54,685 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f19600bc, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:54,787 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f19600bc, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:54,787 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=224}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6ec87dd0 to 127.0.0.1:63605 2024-12-09T03:20:54,787 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=224}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:54,787 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=224}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T03:20:54,789 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=224}] zookeeper.ZKUtil(111): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:54,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:54,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:54,791 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:54,791 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=224}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:54,791 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=224}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=224 2024-12-09T03:20:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=224 2024-12-09T03:20:54,793 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:54,794 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=224, resume processing ppid=223 2024-12-09T03:20:54,794 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=224, ppid=223, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 261 msec 2024-12-09T03:20:54,797 DEBUG [PEWorker-4 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:54,797 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T03:20:54,798 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=223, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 270 msec 2024-12-09T03:20:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=223 2024-12-09T03:20:54,847 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:54,847 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=225, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=225 2024-12-09T03:20:54,848 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:54,849 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=225, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:54,850 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=225, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-09T03:20:54,850 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:54,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=225 2024-12-09T03:20:54,957 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T03:20:54,957 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:54,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=226, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:54,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=226 2024-12-09T03:20:54,959 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:54,960 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=226, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:54,961 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=226, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T03:20:54,961 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:55,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=226 2024-12-09T03:20:55,067 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:55,079 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[1] Thread=217 (was 217), OpenFileDescriptor=448 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=491 (was 491), ProcessCount=11 (was 11), AvailableMemoryMB=9175 (was 9187) 2024-12-09T03:20:55,088 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[1] Thread=217, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=491, ProcessCount=11, AvailableMemoryMB=9175 2024-12-09T03:20:55,089 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=dummy_peer 2024-12-09T03:20:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=227, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:55,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=227 2024-12-09T03:20:55,091 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer dummy_peer, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:55,091 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=227, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:55,092 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=227, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T03:20:55,092 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:55,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=227 2024-12-09T03:20:55,197 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: dummy_peer failed with dummy_peer 2024-12-09T03:20:55,197 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:55,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=228, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:55,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=228 2024-12-09T03:20:55,199 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:55,199 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=228, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:55,200 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=228, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-09T03:20:55,200 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:55,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=228 2024-12-09T03:20:55,307 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T03:20:55,308 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:55,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=229, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:55,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=229 2024-12-09T03:20:55,309 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:55,310 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=229, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:55,310 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=229, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-09T03:20:55,310 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:55,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=229 2024-12-09T03:20:55,417 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:55,429 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[1] Thread=217 (was 217), OpenFileDescriptor=448 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=491 (was 491), ProcessCount=11 (was 11), AvailableMemoryMB=9174 (was 9175) 2024-12-09T03:20:55,438 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerConfig[1] Thread=217, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=491, ProcessCount=11, AvailableMemoryMB=9174 2024-12-09T03:20:55,439 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=230, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:55,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=230 2024-12-09T03:20:55,455 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:55,455 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:55,460 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:55,460 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f19600bd connected 2024-12-09T03:20:55,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=230 2024-12-09T03:20:55,562 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f19600bd, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:55,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=231, ppid=230, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:55,718 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:55,718 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=231 2024-12-09T03:20:55,718 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T03:20:55,734 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=231}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:55,747 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:55,754 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=231}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=231 2024-12-09T03:20:55,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=231 2024-12-09T03:20:55,756 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:55,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=230 2024-12-09T03:20:55,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=231, resume processing ppid=230 2024-12-09T03:20:55,757 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=231, ppid=230, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 190 msec 2024-12-09T03:20:55,757 INFO [PEWorker-2 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T03:20:55,759 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=230, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 318 msec 2024-12-09T03:20:55,786 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x74ffd08b to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@505cce34 2024-12-09T03:20:55,790 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:55,790 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:55,790 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@36b8c5d4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:55,792 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:55,800 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:55,800 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:55,800 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f19600bf connected 2024-12-09T03:20:56,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=230 2024-12-09T03:20:56,067 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:56,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-12-09T03:20:56,068 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:56,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=232, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:56,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=232 2024-12-09T03:20:56,071 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:56,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:56,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:56,073 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:56,073 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=233, ppid=232, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:56,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=232 2024-12-09T03:20:56,224 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:56,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=233 2024-12-09T03:20:56,225 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=233}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T03:20:56,225 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=233}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T03:20:56,225 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=233}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T03:20:56,225 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:56,225 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f19600bf, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:56,225 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f19600bf, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:56,327 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f19600bf, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:56,327 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=233}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x74ffd08b to 127.0.0.1:63605 2024-12-09T03:20:56,327 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=233}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:56,328 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=233}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T03:20:56,329 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=233}] zookeeper.ZKUtil(111): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:56,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:56,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:56,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:56,332 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=233}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:56,332 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=233}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=233 2024-12-09T03:20:56,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=233 2024-12-09T03:20:56,334 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:56,335 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=233, resume processing ppid=232 2024-12-09T03:20:56,335 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=233, ppid=232, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 261 msec 2024-12-09T03:20:56,338 DEBUG [PEWorker-1 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:56,338 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T03:20:56,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=232, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 270 msec 2024-12-09T03:20:56,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=232 2024-12-09T03:20:56,387 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:56,387 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:56,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=234, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:56,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=234 2024-12-09T03:20:56,389 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:56,390 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=234, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:56,390 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=234, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T03:20:56,390 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=234 2024-12-09T03:20:56,497 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T03:20:56,497 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:56,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=235, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:56,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=235 2024-12-09T03:20:56,499 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:56,500 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=235, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:56,500 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=235, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T03:20:56,500 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:56,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=235 2024-12-09T03:20:56,607 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:56,619 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerConfig[1] Thread=217 (was 217), OpenFileDescriptor=448 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=491 (was 491), ProcessCount=11 (was 11), AvailableMemoryMB=9168 (was 9174) 2024-12-09T03:20:56,628 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[1] Thread=217, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=491, ProcessCount=11, AvailableMemoryMB=9168 2024-12-09T03:20:56,630 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-09T03:20:56,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] ipc.CallRunner(138): callId: 752 service: MasterService methodName: AddReplicationPeer size: 80 connection: 172.17.0.2:39754 deadline: 1733714516630, exception=java.io.IOException: Replication peer modification disabled 2024-12-09T03:20:56,631 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(168): Call to master failed, tries = 1, maxAttempts = 2, timeout = 120000 ms, time elapsed = 1 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:245) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:431) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 33 more 2024-12-09T03:20:56,736 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-12-09T03:20:56,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] ipc.CallRunner(138): callId: 753 service: MasterService methodName: AddReplicationPeer size: 80 connection: 172.17.0.2:39754 deadline: 1733714516736, exception=java.io.IOException: Replication peer modification disabled 2024-12-09T03:20:56,737 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(168): Call to master failed, tries = 2, maxAttempts = 2, timeout = 120000 ms, time elapsed = 107 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:245) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:431) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 33 more 2024-12-09T03:20:56,738 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Failed after attempts=2, exceptions: 2024-12-09T03:20:56.631Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) 2024-12-09T03:20:56.738Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) 2024-12-09T03:20:56,740 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:56,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=236, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:56,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=236 2024-12-09T03:20:56,741 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:56,742 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=236, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:56,742 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=236, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-09T03:20:56,742 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:56,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=236 2024-12-09T03:20:56,847 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T03:20:56,848 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:56,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=237, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:56,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=237 2024-12-09T03:20:56,849 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:56,850 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=237, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:56,850 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=237, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-09T03:20:56,850 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:56,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=237 2024-12-09T03:20:56,957 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:56,969 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[1] Thread=217 (was 217), OpenFileDescriptor=448 (was 448), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=491 (was 491), ProcessCount=11 (was 11), AvailableMemoryMB=9165 (was 9168) 2024-12-09T03:20:56,978 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[1] Thread=217, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=491, ProcessCount=11, AvailableMemoryMB=9165 2024-12-09T03:20:56,978 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:56,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=238, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:56,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=238 2024-12-09T03:20:56,994 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:56,994 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:56,997 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:56,997 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f19600c0 connected 2024-12-09T03:20:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=238 2024-12-09T03:20:57,099 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f19600c0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:57,103 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=239, ppid=238, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:57,255 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:57,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=239 2024-12-09T03:20:57,255 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T03:20:57,270 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=239}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:57,283 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:57,290 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=239}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=239 2024-12-09T03:20:57,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=239 2024-12-09T03:20:57,292 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:57,293 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=239, resume processing ppid=238 2024-12-09T03:20:57,293 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=239, ppid=238, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 189 msec 2024-12-09T03:20:57,293 INFO [PEWorker-5 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T03:20:57,294 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=238, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 315 msec 2024-12-09T03:20:57,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=238 2024-12-09T03:20:57,297 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:57,297 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:57,298 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:57,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=240, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=240 2024-12-09T03:20:57,300 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:57,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:57,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:57,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:57,302 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=241, ppid=240, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:57,318 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5f0e337a to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@70d45f21 2024-12-09T03:20:57,320 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:57,321 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:57,321 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@13234c94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:57,322 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:57,324 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:57,324 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:57,327 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f19600c2 connected 2024-12-09T03:20:57,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=240 2024-12-09T03:20:57,453 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:57,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=241 2024-12-09T03:20:57,454 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=241}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T03:20:57,454 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=241}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T03:20:57,454 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=241}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T03:20:57,454 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:57,455 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f19600c2, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:57,455 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f19600c2, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:57,557 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f19600c2, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:57,557 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=241}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5f0e337a to 127.0.0.1:63605 2024-12-09T03:20:57,557 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=241}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:57,557 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=241}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T03:20:57,559 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=241}] zookeeper.ZKUtil(111): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:57,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:57,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:57,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:57,561 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=241}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:57,561 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=241}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=241 2024-12-09T03:20:57,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=241 2024-12-09T03:20:57,564 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:57,565 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=241, resume processing ppid=240 2024-12-09T03:20:57,565 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=241, ppid=240, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 262 msec 2024-12-09T03:20:57,568 DEBUG [PEWorker-4 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:57,568 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T03:20:57,569 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=240, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 270 msec 2024-12-09T03:20:57,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=240 2024-12-09T03:20:57,618 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:57,619 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:57,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=242, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:57,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=242 2024-12-09T03:20:57,620 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:57,621 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=242, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:57,622 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=242, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=2 msec 2024-12-09T03:20:57,622 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:57,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=242 2024-12-09T03:20:57,728 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:57,739 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[1] Thread=219 (was 217) - Thread LEAK? -, OpenFileDescriptor=450 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=491 (was 491), ProcessCount=11 (was 11), AvailableMemoryMB=9169 (was 9165) - AvailableMemoryMB LEAK? - 2024-12-09T03:20:57,749 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerBandwidth[1] Thread=219, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=491, ProcessCount=11, AvailableMemoryMB=9168 2024-12-09T03:20:57,750 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:57,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=243, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:57,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=243 2024-12-09T03:20:57,767 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:57,767 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:57,770 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:57,770 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f19600c3 connected 2024-12-09T03:20:57,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=243 2024-12-09T03:20:57,871 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f19600c3, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:57,878 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=244, ppid=243, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:58,030 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:58,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=244 2024-12-09T03:20:58,031 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=244}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T03:20:58,046 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=244}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:58,059 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=244}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:58,065 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=244}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=244 2024-12-09T03:20:58,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=244 2024-12-09T03:20:58,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=243 2024-12-09T03:20:58,066 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:58,068 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=244, resume processing ppid=243 2024-12-09T03:20:58,068 INFO [PEWorker-4 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T03:20:58,068 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=244, ppid=243, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 188 msec 2024-12-09T03:20:58,069 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=243, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 318 msec 2024-12-09T03:20:58,092 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3d8e2c5a to 127.0.0.1:63605 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@f1f24b7 2024-12-09T03:20:58,095 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-12-09T03:20:58,095 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-12-09T03:20:58,095 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@56289d9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-12-09T03:20:58,096 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:58,098 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:58,098 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-12-09T03:20:58,099 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x10074c9f19600c5 connected 2024-12-09T03:20:58,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=243 2024-12-09T03:20:58,377 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:58,377 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:58,377 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-12-09T03:20:58,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=245, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-12-09T03:20:58,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=245 2024-12-09T03:20:58,393 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-12-09T03:20:58,394 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:63605 2024-12-09T03:20:58,396 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id0x0, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-12-09T03:20:58,396 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f19600c6 connected 2024-12-09T03:20:58,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=245 2024-12-09T03:20:58,497 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@28d9bd0echeck-peer-cluster-id-0x10074c9f19600c6, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:58,500 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=246, ppid=245, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:58,651 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:58,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=246 2024-12-09T03:20:58,652 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-12-09T03:20:58,653 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=246}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=246 2024-12-09T03:20:58,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=246 2024-12-09T03:20:58,654 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:58,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=246, resume processing ppid=245 2024-12-09T03:20:58,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=246, ppid=245, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 154 msec 2024-12-09T03:20:58,655 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:63605:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-12-09T03:20:58,656 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=245, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 277 msec 2024-12-09T03:20:58,676 DEBUG [master/66b6cc25935e:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region 0f8f456dea7eb3922ff1bb2b905c7aee changed from -1.0 to 0.0, refreshing cache 2024-12-09T03:20:58,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=245 2024-12-09T03:20:58,697 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-12-09T03:20:58,697 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-12-09T03:20:58,698 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:58,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=247, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:58,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=247 2024-12-09T03:20:58,700 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:58,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:58,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:58,702 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:58,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=248, ppid=247, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:58,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=247 2024-12-09T03:20:58,853 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:58,854 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=248 2024-12-09T03:20:58,854 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=248}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T03:20:58,854 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=248}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T03:20:58,854 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=248}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T03:20:58,855 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-12-09T03:20:58,855 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x10074c9f19600c5, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:58,855 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x10074c9f19600c5, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-12-09T03:20:58,962 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x10074c9f19600c5, quorum=127.0.0.1:63605, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:20:58,962 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=248}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3d8e2c5a to 127.0.0.1:63605 2024-12-09T03:20:58,962 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=248}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:20:58,962 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=248}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T03:20:58,966 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=248}] zookeeper.ZKUtil(111): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:58,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:20:58,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:58,968 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:20:58,968 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=248}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:58,969 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=248}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=248 2024-12-09T03:20:58,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=248 2024-12-09T03:20:58,976 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:58,977 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=248, resume processing ppid=247 2024-12-09T03:20:58,977 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=248, ppid=247, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 274 msec 2024-12-09T03:20:58,981 DEBUG [PEWorker-1 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:20:58,981 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T03:20:58,983 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=247, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 283 msec 2024-12-09T03:20:59,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=247 2024-12-09T03:20:59,018 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:59,019 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:59,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=249, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:59,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=249 2024-12-09T03:20:59,021 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:59,022 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=249, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:59,025 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=249, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T03:20:59,025 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:59,089 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T03:20:59,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=249 2024-12-09T03:20:59,127 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-12-09T03:20:59,127 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:20:59,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=250, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:59,130 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:20:59,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=250 2024-12-09T03:20:59,130 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=250, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:59,131 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=250, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-12-09T03:20:59,131 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:59,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=250 2024-12-09T03:20:59,238 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:20:59,251 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerBandwidth[1] Thread=220 (was 219) Potentially hanging thread: region-location-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=450 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=468 (was 491), ProcessCount=11 (was 11), AvailableMemoryMB=9157 (was 9168) 2024-12-09T03:20:59,263 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[1] Thread=220, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=468, ProcessCount=11, AvailableMemoryMB=9157 2024-12-09T03:20:59,264 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:59,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=251, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:59,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=251 2024-12-09T03:20:59,272 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=252, ppid=251, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:59,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=251 2024-12-09T03:20:59,423 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:59,424 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=252 2024-12-09T03:20:59,424 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=252}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-12-09T03:20:59,468 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=252}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-12-09T03:20:59,477 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=252}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=252 2024-12-09T03:20:59,477 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(587): peerId=1, queueId=1 (queues=1) is replicating from cluster=3b7d4240-16c9-429e-aa98-985bf779f27b to cluster=3b7d4240-16c9-429e-aa98-985bf779f27b 2024-12-09T03:20:59,477 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSource(343): peerId=1, starting shipping worker for walGroupId=66b6cc25935e%2C45493%2C1733714394961 2024-12-09T03:20:59,477 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1 {}] regionserver.ReplicationSourceWALReader(112): peerClusterZnode=1, ReplicationSourceWALReaderThread : 1 inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-12-09T03:20:59,478 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1.replicationSource.wal-reader.66b6cc25935e%2C45493%2C1733714394961,1 {}] regionserver.WALEntryStream(260): Creating new reader hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/WALs/66b6cc25935e,45493,1733714394961/66b6cc25935e%2C45493%2C1733714394961.1733714397367, startPosition=0, beingWritten=true 2024-12-09T03:20:59,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=252 2024-12-09T03:20:59,483 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1.replicationSource.shipper66b6cc25935e%2C45493%2C1733714394961,1 {}] regionserver.ReplicationSourceShipper(98): Running ReplicationSourceShipper Thread for wal group: 66b6cc25935e%2C45493%2C1733714394961 2024-12-09T03:20:59,483 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:20:59,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=252, resume processing ppid=251 2024-12-09T03:20:59,489 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=252, ppid=251, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 211 msec 2024-12-09T03:20:59,489 INFO [PEWorker-1 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false 2024-12-09T03:20:59,490 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=251, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 225 msec 2024-12-09T03:20:59,495 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1.replicationSource.wal-reader.66b6cc25935e%2C45493%2C1733714394961,1 {}] regionserver.ReplicationSourceWALReader(181): Read 0 WAL entries eligible for replication 2024-12-09T03:20:59,495 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1.replicationSource.wal-reader.66b6cc25935e%2C45493%2C1733714394961,1 {}] regionserver.WALEntryStream(223): Reset reader hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/WALs/66b6cc25935e,45493,1733714394961/66b6cc25935e%2C45493%2C1733714394961.1733714397367 to pos 589, reset compression=false 2024-12-09T03:20:59,495 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1.replicationSource.shipper66b6cc25935e%2C45493%2C1733714394961,1 {}] regionserver.ReplicationSourceShipper(110): Shipper from source 1 got entry batch from reader: WALEntryBatch [walEntries=[], lastWalPath=hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/WALs/66b6cc25935e,45493,1733714394961/66b6cc25935e%2C45493%2C1733714394961.1733714397367, lastWalPosition=589, nbRowKeys=0, nbHFiles=0, heapSize=0, lastSeqIds={}, endOfFile=false,usedBufferSize=0] 2024-12-09T03:20:59,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=251 2024-12-09T03:20:59,587 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:20:59,587 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=2, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-12-09T03:20:59,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=253, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-12-09T03:20:59,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=253 2024-12-09T03:20:59,589 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] ... 13 more 2024-12-09T03:20:59,590 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=253, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:20:59,591 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=253, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=3 msec 2024-12-09T03:20:59,591 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:20:59,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=253 2024-12-09T03:20:59,697 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 2 failed with Invalid cluster key: 2024-12-09T03:20:59,698 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-12-09T03:20:59,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=254, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:20:59,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-12-09T03:20:59,708 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:59,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-12-09T03:20:59,711 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:59,712 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-12-09T03:20:59,712 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=255, ppid=254, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-12-09T03:20:59,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-12-09T03:20:59,865 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 66b6cc25935e,45493,1733714394961 2024-12-09T03:20:59,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45493 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=255 2024-12-09T03:20:59,865 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=255}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-12-09T03:20:59,866 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=255}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-12-09T03:20:59,866 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=255}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-12-09T03:21:00,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-12-09T03:21:00,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-12-09T03:21:00,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-12-09T03:21:00,866 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1.replicationSource.wal-reader.66b6cc25935e%2C45493%2C1733714394961,1 {}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:131) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:166) ~[classes/:?] 2024-12-09T03:21:00,866 WARN [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1.replicationSource.shipper66b6cc25935e%2C45493%2C1733714394961,1 {}] regionserver.ReplicationSourceShipper(123): Interrupted while waiting for next replication entry batch java.lang.InterruptedException: null at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1681) ~[?:?] at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) ~[?:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.poll(ReplicationSourceWALReader.java:313) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceShipper.run(ReplicationSourceShipper.java:109) ~[classes/:?] 2024-12-09T03:21:01,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-12-09T03:21:01,866 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=255}] regionserver.ReplicationSource(711): peerId=1, ReplicationSourceWorker RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0.replicationSource,1.replicationSource.shipper66b6cc25935e%2C45493%2C1733714394961,1 terminated 2024-12-09T03:21:01,866 INFO [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=255}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-12-09T03:21:01,867 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=255}] zookeeper.ZKUtil(111): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:21:01,869 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1/66b6cc25935e%2C45493%2C1733714394961.1733714397367 2024-12-09T03:21:01,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:21:01,870 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/66b6cc25935e,45493,1733714394961/1 2024-12-09T03:21:01,870 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=255}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:21:01,870 DEBUG [RS_REFRESH_PEER-regionserver/66b6cc25935e:0-0 {event_type=RS_REFRESH_PEER, pid=255}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=255 2024-12-09T03:21:01,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(4106): Remote procedure done, pid=255 2024-12-09T03:21:01,872 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 66b6cc25935e,45493,1733714394961 suceeded 2024-12-09T03:21:01,873 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=255, resume processing ppid=254 2024-12-09T03:21:01,873 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=255, ppid=254, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 2.1600 sec 2024-12-09T03:21:01,876 DEBUG [PEWorker-2 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-12-09T03:21:01,876 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-12-09T03:21:01,877 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=254, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 2.1780 sec 2024-12-09T03:21:03,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=254 2024-12-09T03:21:03,867 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-12-09T03:21:03,867 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-12-09T03:21:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] procedure2.ProcedureExecutor(1098): Stored pid=256, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-12-09T03:21:03,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=256 2024-12-09T03:21:03,869 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-12-09T03:21:03,870 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=256, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-12-09T03:21:03,870 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=256, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-12-09T03:21:03,870 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-12-09T03:21:03,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45333 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=256 2024-12-09T03:21:03,984 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-12-09T03:21:04,000 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[1] Thread=215 (was 220), OpenFileDescriptor=445 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=478 (was 468) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=9153 (was 9157) 2024-12-09T03:21:04,001 INFO [Time-limited test {}] client.AsyncConnectionImpl(225): Connection has been closed by Time-limited test. 2024-12-09T03:21:04,001 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(253): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:227) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:219) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.client.TestAsyncAdminBase.tearDownAfterClass(TestAsyncAdminBase.java:96) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-12-09T03:21:04,001 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:21:04,002 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0e81966d to 127.0.0.1:63605 2024-12-09T03:21:04,002 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1340): Shutting down minicluster 2024-12-09T03:21:04,002 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x005c3638 to 127.0.0.1:63605 2024-12-09T03:21:04,002 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:21:04,003 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-12-09T03:21:04,003 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=956768932, stopped=false 2024-12-09T03:21:04,003 INFO [Time-limited test {}] master.ServerManager(987): Cluster shutdown requested of master=66b6cc25935e,45333,1733714394200 2024-12-09T03:21:04,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T03:21:04,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:21:04,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-12-09T03:21:04,010 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:21:04,011 INFO [Time-limited test {}] procedure2.ProcedureExecutor(700): Stopping 2024-12-09T03:21:04,011 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:21:04,011 INFO [Time-limited test {}] regionserver.HRegionServer(2561): ***** STOPPING region server '66b6cc25935e,45493,1733714394961' ***** 2024-12-09T03:21:04,011 INFO [Time-limited test {}] regionserver.HRegionServer(2575): STOPPED: Shutdown requested 2024-12-09T03:21:04,011 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T03:21:04,011 INFO [RS:0;66b6cc25935e:45493 {}] regionserver.HeapMemoryManager(220): Stopping 2024-12-09T03:21:04,011 INFO [RS:0;66b6cc25935e:45493 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-12-09T03:21:04,012 INFO [RS:0;66b6cc25935e:45493 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-12-09T03:21:04,012 INFO [RS:0;66b6cc25935e:45493 {}] regionserver.HRegionServer(3579): Received CLOSE for 0f8f456dea7eb3922ff1bb2b905c7aee 2024-12-09T03:21:04,012 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-12-09T03:21:04,012 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(373): MemStoreFlusher.0 exiting 2024-12-09T03:21:04,020 INFO [RS:0;66b6cc25935e:45493 {}] regionserver.HRegionServer(1224): stopping server 66b6cc25935e,45493,1733714394961 2024-12-09T03:21:04,020 DEBUG [RS:0;66b6cc25935e:45493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:21:04,020 INFO [RS:0;66b6cc25935e:45493 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-12-09T03:21:04,020 INFO [RS:0;66b6cc25935e:45493 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-12-09T03:21:04,020 INFO [RS:0;66b6cc25935e:45493 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-12-09T03:21:04,020 INFO [RS:0;66b6cc25935e:45493 {}] regionserver.HRegionServer(3579): Received CLOSE for 1588230740 2024-12-09T03:21:04,021 INFO [RS:0;66b6cc25935e:45493 {}] regionserver.HRegionServer(1599): Waiting on 2 regions to close 2024-12-09T03:21:04,021 DEBUG [RS:0;66b6cc25935e:45493 {}] regionserver.HRegionServer(1603): Online Regions={0f8f456dea7eb3922ff1bb2b905c7aee=hbase:namespace,,1733714398605.0f8f456dea7eb3922ff1bb2b905c7aee., 1588230740=hbase:meta,,1.1588230740} 2024-12-09T03:21:04,021 DEBUG [RS:0;66b6cc25935e:45493 {}] regionserver.HRegionServer(1629): Waiting on 0f8f456dea7eb3922ff1bb2b905c7aee, 1588230740 2024-12-09T03:21:04,022 DEBUG [RS_CLOSE_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-12-09T03:21:04,022 INFO [RS_CLOSE_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-12-09T03:21:04,022 DEBUG [RS_CLOSE_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-12-09T03:21:04,022 DEBUG [RS_CLOSE_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-12-09T03:21:04,022 DEBUG [RS_CLOSE_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-12-09T03:21:04,023 INFO [RS_CLOSE_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=1.23 KB heapSize=2.87 KB 2024-12-09T03:21:04,026 DEBUG [RS_CLOSE_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1681): Closing 0f8f456dea7eb3922ff1bb2b905c7aee, disabling compactions & flushes 2024-12-09T03:21:04,027 INFO [RS_CLOSE_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1703): Closing region hbase:namespace,,1733714398605.0f8f456dea7eb3922ff1bb2b905c7aee. 2024-12-09T03:21:04,027 DEBUG [RS_CLOSE_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1733714398605.0f8f456dea7eb3922ff1bb2b905c7aee. 2024-12-09T03:21:04,027 DEBUG [RS_CLOSE_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1733714398605.0f8f456dea7eb3922ff1bb2b905c7aee. after waiting 0 ms 2024-12-09T03:21:04,027 DEBUG [RS_CLOSE_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1733714398605.0f8f456dea7eb3922ff1bb2b905c7aee. 2024-12-09T03:21:04,027 INFO [RS_CLOSE_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(2837): Flushing 0f8f456dea7eb3922ff1bb2b905c7aee 1/1 column families, dataSize=78 B heapSize=488 B 2024-12-09T03:21:04,048 INFO [regionserver/66b6cc25935e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T03:21:04,144 DEBUG [RS_CLOSE_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_REGION}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/data/hbase/namespace/0f8f456dea7eb3922ff1bb2b905c7aee/.tmp/info/463c13bbdf1149c3b475e5f38eb7301c is 45, key is default/info:d/1733714399780/Put/seqid=0 2024-12-09T03:21:04,145 DEBUG [RS_CLOSE_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/data/hbase/meta/1588230740/.tmp/info/eef930f356f343c8adaa9c66fc314189 is 143, key is hbase:namespace,,1733714398605.0f8f456dea7eb3922ff1bb2b905c7aee./info:regioninfo/1733714399683/Put/seqid=0 2024-12-09T03:21:04,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43681 is added to blk_1073741837_1013 (size=5037) 2024-12-09T03:21:04,168 INFO [RS_CLOSE_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/data/hbase/namespace/0f8f456dea7eb3922ff1bb2b905c7aee/.tmp/info/463c13bbdf1149c3b475e5f38eb7301c 2024-12-09T03:21:04,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43681 is added to blk_1073741838_1014 (size=6595) 2024-12-09T03:21:04,176 INFO [RS_CLOSE_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.14 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/data/hbase/meta/1588230740/.tmp/info/eef930f356f343c8adaa9c66fc314189 2024-12-09T03:21:04,185 INFO [regionserver/66b6cc25935e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-12-09T03:21:04,185 INFO [regionserver/66b6cc25935e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-12-09T03:21:04,222 DEBUG [RS:0;66b6cc25935e:45493 {}] regionserver.HRegionServer(1629): Waiting on 0f8f456dea7eb3922ff1bb2b905c7aee, 1588230740 2024-12-09T03:21:04,255 DEBUG [RS_CLOSE_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/data/hbase/namespace/0f8f456dea7eb3922ff1bb2b905c7aee/.tmp/info/463c13bbdf1149c3b475e5f38eb7301c as hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/data/hbase/namespace/0f8f456dea7eb3922ff1bb2b905c7aee/info/463c13bbdf1149c3b475e5f38eb7301c 2024-12-09T03:21:04,266 INFO [RS_CLOSE_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/data/hbase/namespace/0f8f456dea7eb3922ff1bb2b905c7aee/info/463c13bbdf1149c3b475e5f38eb7301c, entries=2, sequenceid=6, filesize=4.9 K 2024-12-09T03:21:04,269 INFO [RS_CLOSE_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for 0f8f456dea7eb3922ff1bb2b905c7aee in 242ms, sequenceid=6, compaction requested=false 2024-12-09T03:21:04,277 DEBUG [RS_CLOSE_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/data/hbase/meta/1588230740/.tmp/table/3eafd98ff2ce4f3bb3b503b0a551eb1a is 51, key is hbase:namespace/table:state/1733714399705/Put/seqid=0 2024-12-09T03:21:04,301 DEBUG [RS_CLOSE_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_REGION}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/data/hbase/namespace/0f8f456dea7eb3922ff1bb2b905c7aee/recovered.edits/9.seqid, newMaxSeqId=9, maxSeqId=1 2024-12-09T03:21:04,312 INFO [RS_CLOSE_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1922): Closed hbase:namespace,,1733714398605.0f8f456dea7eb3922ff1bb2b905c7aee. 2024-12-09T03:21:04,313 DEBUG [RS_CLOSE_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_REGION}] regionserver.HRegion(1635): Region close journal for 0f8f456dea7eb3922ff1bb2b905c7aee: 2024-12-09T03:21:04,313 DEBUG [RS_CLOSE_REGION-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_REGION}] handler.CloseRegionHandler(117): Closed hbase:namespace,,1733714398605.0f8f456dea7eb3922ff1bb2b905c7aee. 2024-12-09T03:21:04,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43681 is added to blk_1073741839_1015 (size=5242) 2024-12-09T03:21:04,318 INFO [RS_CLOSE_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=94 B at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/data/hbase/meta/1588230740/.tmp/table/3eafd98ff2ce4f3bb3b503b0a551eb1a 2024-12-09T03:21:04,332 DEBUG [RS_CLOSE_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/data/hbase/meta/1588230740/.tmp/info/eef930f356f343c8adaa9c66fc314189 as hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/data/hbase/meta/1588230740/info/eef930f356f343c8adaa9c66fc314189 2024-12-09T03:21:04,347 INFO [RS_CLOSE_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/data/hbase/meta/1588230740/info/eef930f356f343c8adaa9c66fc314189, entries=10, sequenceid=9, filesize=6.4 K 2024-12-09T03:21:04,350 DEBUG [RS_CLOSE_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/data/hbase/meta/1588230740/.tmp/table/3eafd98ff2ce4f3bb3b503b0a551eb1a as hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/data/hbase/meta/1588230740/table/3eafd98ff2ce4f3bb3b503b0a551eb1a 2024-12-09T03:21:04,361 INFO [RS_CLOSE_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/data/hbase/meta/1588230740/table/3eafd98ff2ce4f3bb3b503b0a551eb1a, entries=2, sequenceid=9, filesize=5.1 K 2024-12-09T03:21:04,363 INFO [RS_CLOSE_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3040): Finished flush of dataSize ~1.23 KB/1264, heapSize ~2.59 KB/2648, currentSize=0 B/0 for 1588230740 in 340ms, sequenceid=9, compaction requested=false 2024-12-09T03:21:04,396 DEBUG [RS_CLOSE_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/data/hbase/meta/1588230740/recovered.edits/12.seqid, newMaxSeqId=12, maxSeqId=1 2024-12-09T03:21:04,398 DEBUG [RS_CLOSE_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T03:21:04,398 INFO [RS_CLOSE_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-12-09T03:21:04,398 DEBUG [RS_CLOSE_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-12-09T03:21:04,399 DEBUG [RS_CLOSE_META-regionserver/66b6cc25935e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-12-09T03:21:04,422 INFO [RS:0;66b6cc25935e:45493 {}] regionserver.HRegionServer(1250): stopping server 66b6cc25935e,45493,1733714394961; all regions closed. 2024-12-09T03:21:04,430 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43681 is added to blk_1073741834_1010 (size=2484) 2024-12-09T03:21:04,440 DEBUG [RS:0;66b6cc25935e:45493 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/oldWALs 2024-12-09T03:21:04,440 INFO [RS:0;66b6cc25935e:45493 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL 66b6cc25935e%2C45493%2C1733714394961.meta:.meta(num 1733714398338) 2024-12-09T03:21:04,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43681 is added to blk_1073741832_1008 (size=1414) 2024-12-09T03:21:04,450 DEBUG [RS:0;66b6cc25935e:45493 {}] wal.AbstractFSWAL(1071): Moved 1 WAL file(s) to /user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/oldWALs 2024-12-09T03:21:04,450 INFO [RS:0;66b6cc25935e:45493 {}] wal.AbstractFSWAL(1074): Closed WAL: AsyncFSWAL 66b6cc25935e%2C45493%2C1733714394961:(num 1733714397367) 2024-12-09T03:21:04,450 DEBUG [RS:0;66b6cc25935e:45493 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:21:04,450 INFO [RS:0;66b6cc25935e:45493 {}] regionserver.LeaseManager(133): Closed leases 2024-12-09T03:21:04,450 INFO [RS:0;66b6cc25935e:45493 {}] hbase.ChoreService(370): Chore service for: regionserver/66b6cc25935e:0 had [ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS] on shutdown 2024-12-09T03:21:04,451 INFO [RS:0;66b6cc25935e:45493 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:45493 2024-12-09T03:21:04,452 INFO [regionserver/66b6cc25935e:0.logRoller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-09T03:21:04,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-12-09T03:21:04,456 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/66b6cc25935e,45493,1733714394961 2024-12-09T03:21:04,457 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [66b6cc25935e,45493,1733714394961] 2024-12-09T03:21:04,457 DEBUG [RegionServerTracker-0 {}] master.DeadServer(103): Processing 66b6cc25935e,45493,1733714394961; numProcessing=1 2024-12-09T03:21:04,459 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/draining/66b6cc25935e,45493,1733714394961 already deleted, retry=false 2024-12-09T03:21:04,459 INFO [RegionServerTracker-0 {}] master.ServerManager(652): Cluster shutdown set; 66b6cc25935e,45493,1733714394961 expired; onlineServers=0 2024-12-09T03:21:04,459 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2561): ***** STOPPING region server '66b6cc25935e,45333,1733714394200' ***** 2024-12-09T03:21:04,459 INFO [RegionServerTracker-0 {}] regionserver.HRegionServer(2575): STOPPED: Cluster shutdown set; onlineServer=0 2024-12-09T03:21:04,459 DEBUG [M:0;66b6cc25935e:45333 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6abf1a6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=66b6cc25935e/172.17.0.2:0 2024-12-09T03:21:04,459 INFO [M:0;66b6cc25935e:45333 {}] regionserver.HRegionServer(1224): stopping server 66b6cc25935e,45333,1733714394200 2024-12-09T03:21:04,459 INFO [M:0;66b6cc25935e:45333 {}] regionserver.HRegionServer(1250): stopping server 66b6cc25935e,45333,1733714394200; all regions closed. 2024-12-09T03:21:04,459 DEBUG [M:0;66b6cc25935e:45333 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-12-09T03:21:04,460 DEBUG [M:0;66b6cc25935e:45333 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-12-09T03:21:04,460 DEBUG [M:0;66b6cc25935e:45333 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-12-09T03:21:04,460 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-12-09T03:21:04,460 DEBUG [master/66b6cc25935e:0:becomeActiveMaster-HFileCleaner.large.0-1733714397086 {}] cleaner.HFileCleaner(306): Exit Thread[master/66b6cc25935e:0:becomeActiveMaster-HFileCleaner.large.0-1733714397086,5,FailOnTimeoutGroup] 2024-12-09T03:21:04,460 DEBUG [master/66b6cc25935e:0:becomeActiveMaster-HFileCleaner.small.0-1733714397086 {}] cleaner.HFileCleaner(306): Exit Thread[master/66b6cc25935e:0:becomeActiveMaster-HFileCleaner.small.0-1733714397086,5,FailOnTimeoutGroup] 2024-12-09T03:21:04,460 INFO [M:0;66b6cc25935e:45333 {}] hbase.ChoreService(370): Chore service for: master/66b6cc25935e:0 had [] on shutdown 2024-12-09T03:21:04,460 DEBUG [M:0;66b6cc25935e:45333 {}] master.HMaster(1733): Stopping service threads 2024-12-09T03:21:04,460 INFO [M:0;66b6cc25935e:45333 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-12-09T03:21:04,461 INFO [M:0;66b6cc25935e:45333 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-12-09T03:21:04,461 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-12-09T03:21:04,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-12-09T03:21:04,462 DEBUG [M:0;66b6cc25935e:45333 {}] zookeeper.ZKUtil(347): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-12-09T03:21:04,462 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-12-09T03:21:04,462 WARN [M:0;66b6cc25935e:45333 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-12-09T03:21:04,462 INFO [M:0;66b6cc25935e:45333 {}] assignment.AssignmentManager(391): Stopping assignment manager 2024-12-09T03:21:04,462 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-12-09T03:21:04,463 INFO [M:0;66b6cc25935e:45333 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-12-09T03:21:04,463 DEBUG [M:0;66b6cc25935e:45333 {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-12-09T03:21:04,463 INFO [M:0;66b6cc25935e:45333 {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T03:21:04,463 DEBUG [M:0;66b6cc25935e:45333 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T03:21:04,463 DEBUG [M:0;66b6cc25935e:45333 {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-12-09T03:21:04,463 DEBUG [M:0;66b6cc25935e:45333 {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T03:21:04,463 INFO [M:0;66b6cc25935e:45333 {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=750.00 KB heapSize=899.16 KB 2024-12-09T03:21:04,465 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-12-09T03:21:04,498 DEBUG [M:0;66b6cc25935e:45333 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c860985961f9480fa6cc4b3bce4efbd8 is 82, key is hbase:meta,,1/info:regioninfo/1733714398477/Put/seqid=0 2024-12-09T03:21:04,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43681 is added to blk_1073741840_1016 (size=5672) 2024-12-09T03:21:04,513 INFO [M:0;66b6cc25935e:45333 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=1814 (bloomFilter=true), to=hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c860985961f9480fa6cc4b3bce4efbd8 2024-12-09T03:21:04,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:21:04,561 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45493-0x10074c9f1960001, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:21:04,561 INFO [RS:0;66b6cc25935e:45493 {}] regionserver.HRegionServer(1307): Exiting; stopping=66b6cc25935e,45493,1733714394961; zookeeper connection closed. 2024-12-09T03:21:04,561 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@44a92d96 {}] hbase.MiniHBaseCluster$SingleFileSystemShutdownThread(216): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@44a92d96 2024-12-09T03:21:04,562 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-12-09T03:21:04,604 DEBUG [M:0;66b6cc25935e:45333 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0c1e84ca5cad44ada7df04b6035afa12 is 2089, key is \x00\x00\x00\x00\x00\x00\x00\x9D/proc:d/1733714439671/Put/seqid=0 2024-12-09T03:21:04,623 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43681 is added to blk_1073741841_1017 (size=168558) 2024-12-09T03:21:04,630 INFO [M:0;66b6cc25935e:45333 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=747.45 KB at sequenceid=1814 (bloomFilter=true), to=hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0c1e84ca5cad44ada7df04b6035afa12 2024-12-09T03:21:04,661 INFO [M:0;66b6cc25935e:45333 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0c1e84ca5cad44ada7df04b6035afa12 2024-12-09T03:21:04,688 DEBUG [M:0;66b6cc25935e:45333 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9e79c65cd5e94730bf1e9be128fa4052 is 69, key is 66b6cc25935e,45493,1733714394961/rs:state/1733714397099/Put/seqid=0 2024-12-09T03:21:04,703 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-12-09T03:21:04,703 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-12-09T03:21:04,703 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-12-09T03:21:04,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43681 is added to blk_1073741842_1018 (size=5156) 2024-12-09T03:21:05,124 INFO [M:0;66b6cc25935e:45333 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=1814 (bloomFilter=true), to=hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9e79c65cd5e94730bf1e9be128fa4052 2024-12-09T03:21:05,188 DEBUG [M:0;66b6cc25935e:45333 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ccf5414f9f604f48b34ad55628d55ca2 is 68, key is replication_peer_modification_on/state:d/1733714463985/Put/seqid=0 2024-12-09T03:21:05,205 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43681 is added to blk_1073741843_1019 (size=5154) 2024-12-09T03:21:05,209 INFO [M:0;66b6cc25935e:45333 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=2 KB at sequenceid=1814 (bloomFilter=true), to=hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ccf5414f9f604f48b34ad55628d55ca2 2024-12-09T03:21:05,220 DEBUG [M:0;66b6cc25935e:45333 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c860985961f9480fa6cc4b3bce4efbd8 as hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c860985961f9480fa6cc4b3bce4efbd8 2024-12-09T03:21:05,231 INFO [M:0;66b6cc25935e:45333 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c860985961f9480fa6cc4b3bce4efbd8, entries=8, sequenceid=1814, filesize=5.5 K 2024-12-09T03:21:05,232 DEBUG [M:0;66b6cc25935e:45333 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/0c1e84ca5cad44ada7df04b6035afa12 as hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0c1e84ca5cad44ada7df04b6035afa12 2024-12-09T03:21:05,242 INFO [M:0;66b6cc25935e:45333 {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for 0c1e84ca5cad44ada7df04b6035afa12 2024-12-09T03:21:05,242 INFO [M:0;66b6cc25935e:45333 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/0c1e84ca5cad44ada7df04b6035afa12, entries=256, sequenceid=1814, filesize=164.6 K 2024-12-09T03:21:05,244 DEBUG [M:0;66b6cc25935e:45333 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/9e79c65cd5e94730bf1e9be128fa4052 as hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9e79c65cd5e94730bf1e9be128fa4052 2024-12-09T03:21:05,253 INFO [M:0;66b6cc25935e:45333 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/9e79c65cd5e94730bf1e9be128fa4052, entries=1, sequenceid=1814, filesize=5.0 K 2024-12-09T03:21:05,256 DEBUG [M:0;66b6cc25935e:45333 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/ccf5414f9f604f48b34ad55628d55ca2 as hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ccf5414f9f604f48b34ad55628d55ca2 2024-12-09T03:21:05,265 INFO [M:0;66b6cc25935e:45333 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:43899/user/jenkins/test-data/9c546d54-c648-0fa2-df63-0f2dad7fe324/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/ccf5414f9f604f48b34ad55628d55ca2, entries=1, sequenceid=1814, filesize=5.0 K 2024-12-09T03:21:05,266 INFO [M:0;66b6cc25935e:45333 {}] regionserver.HRegion(3040): Finished flush of dataSize ~750.00 KB/768002, heapSize ~899.09 KB/920672, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 803ms, sequenceid=1814, compaction requested=false 2024-12-09T03:21:05,272 INFO [M:0;66b6cc25935e:45333 {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-12-09T03:21:05,272 DEBUG [M:0;66b6cc25935e:45333 {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-12-09T03:21:05,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:43681 is added to blk_1073741830_1006 (size=887137) 2024-12-09T03:21:05,276 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(243): LogRoller exiting. 2024-12-09T03:21:05,276 INFO [M:0;66b6cc25935e:45333 {}] flush.MasterFlushTableProcedureManager(91): stop: server shutting down. 2024-12-09T03:21:05,276 INFO [M:0;66b6cc25935e:45333 {}] ipc.NettyRpcServer(351): Stopping server on /172.17.0.2:45333 2024-12-09T03:21:05,278 DEBUG [M:0;66b6cc25935e:45333 {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/rs/66b6cc25935e,45333,1733714394200 already deleted, retry=false 2024-12-09T03:21:05,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:21:05,384 INFO [M:0;66b6cc25935e:45333 {}] regionserver.HRegionServer(1307): Exiting; stopping=66b6cc25935e,45333,1733714394200; zookeeper connection closed. 2024-12-09T03:21:05,384 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45333-0x10074c9f1960000, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:21:05,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility-0x10074c9f1960019, quorum=127.0.0.1:63605, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-12-09T03:21:05,494 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@a1044e0{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-12-09T03:21:05,497 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@240f5d3{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T03:21:05,497 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T03:21:05,498 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@3045cad6{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T03:21:05,498 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@5148c4c{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/hadoop.log.dir/,STOPPED} 2024-12-09T03:21:05,502 WARN [BP-733451773-172.17.0.2-1733714390664 heartbeating to localhost/127.0.0.1:43899 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-12-09T03:21:05,502 WARN [BP-733451773-172.17.0.2-1733714390664 heartbeating to localhost/127.0.0.1:43899 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-733451773-172.17.0.2-1733714390664 (Datanode Uuid 0a9fb78d-b6f6-407b-a4c8-2f4f21e55997) service to localhost/127.0.0.1:43899 2024-12-09T03:21:05,502 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-12-09T03:21:05,502 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-12-09T03:21:05,503 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/cluster_27f4fc50-796d-9115-8208-caf375f5faf1/dfs/data/data1/current/BP-733451773-172.17.0.2-1733714390664 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T03:21:05,503 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/cluster_27f4fc50-796d-9115-8208-caf375f5faf1/dfs/data/data2/current/BP-733451773-172.17.0.2-1733714390664 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-12-09T03:21:05,503 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-12-09T03:21:05,517 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@2b5eba6b{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-12-09T03:21:05,518 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@79c1d5da{HTTP/1.1, (http/1.1)}{localhost:0} 2024-12-09T03:21:05,519 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-12-09T03:21:05,519 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@f2948e9{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-12-09T03:21:05,519 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@1d9ceadd{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/eed2b3b7-6c8d-8c2d-7c83-a8db62b191de/hadoop.log.dir/,STOPPED} 2024-12-09T03:21:05,538 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(345): Shutdown MiniZK cluster with all ZK servers 2024-12-09T03:21:05,561 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1347): Minicluster is down