2024-11-12 04:08:40,767 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@302c971f 2024-11-12 04:08:40,790 main DEBUG Took 0.019866 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-12 04:08:40,790 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-12 04:08:40,791 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-12 04:08:40,792 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-12 04:08:40,793 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 04:08:40,804 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-12 04:08:40,819 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 04:08:40,821 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 04:08:40,822 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 04:08:40,823 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 04:08:40,824 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 04:08:40,824 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 04:08:40,826 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 04:08:40,826 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 04:08:40,827 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 04:08:40,827 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 04:08:40,829 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 04:08:40,829 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 04:08:40,830 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 04:08:40,831 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 04:08:40,831 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 04:08:40,832 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 04:08:40,833 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 04:08:40,833 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 04:08:40,834 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 04:08:40,834 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 04:08:40,835 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 04:08:40,835 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 04:08:40,836 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 04:08:40,837 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 04:08:40,837 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 04:08:40,838 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-12 04:08:40,840 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 04:08:40,842 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-12 04:08:40,844 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-12 04:08:40,845 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-12 04:08:40,847 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-12 04:08:40,847 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-12 04:08:40,859 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-12 04:08:40,863 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-12 04:08:40,866 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-12 04:08:40,867 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-12 04:08:40,867 main DEBUG createAppenders(={Console}) 2024-11-12 04:08:40,868 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@302c971f initialized 2024-11-12 04:08:40,869 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@302c971f 2024-11-12 04:08:40,869 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@302c971f OK. 2024-11-12 04:08:40,870 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-12 04:08:40,871 main DEBUG OutputStream closed 2024-11-12 04:08:40,871 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-12 04:08:40,871 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-12 04:08:40,872 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@505fc5a4 OK 2024-11-12 04:08:40,975 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-12 04:08:40,978 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-12 04:08:40,979 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-12 04:08:40,981 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-12 04:08:40,981 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-12 04:08:40,982 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-12 04:08:40,982 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-12 04:08:40,983 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-12 04:08:40,983 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-12 04:08:40,984 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-12 04:08:40,984 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-12 04:08:40,985 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-12 04:08:40,985 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-12 04:08:40,986 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-12 04:08:40,986 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-12 04:08:40,986 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-12 04:08:40,987 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-12 04:08:40,988 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-12 04:08:40,991 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12 04:08:40,992 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-logging/target/hbase-logging-2.7.0-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@39de3d36) with optional ClassLoader: null 2024-11-12 04:08:40,992 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-12 04:08:40,993 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@39de3d36] started OK. 2024-11-12T04:08:41,313 DEBUG [main {}] hbase.HBaseTestingUtility(348): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b 2024-11-12 04:08:41,317 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-12 04:08:41,317 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12T04:08:41,360 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.client.TestAsyncReplicationAdminApi timeout: 26 mins 2024-11-12T04:08:41,395 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1126): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-12T04:08:41,399 INFO [Time-limited test {}] hbase.HBaseZKTestingUtility(82): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/cluster_b0a2b79c-cac7-9f39-3098-f01052b75b38, deleteOnExit=true 2024-11-12T04:08:41,400 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1140): STARTING DFS 2024-11-12T04:08:41,401 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/test.cache.data in system properties and HBase conf 2024-11-12T04:08:41,402 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/hadoop.tmp.dir in system properties and HBase conf 2024-11-12T04:08:41,403 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/hadoop.log.dir in system properties and HBase conf 2024-11-12T04:08:41,404 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-12T04:08:41,405 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-12T04:08:41,406 INFO [Time-limited test {}] hbase.HBaseTestingUtility(811): read short circuit is OFF 2024-11-12T04:08:41,563 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-12T04:08:41,693 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T04:08:41,700 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-12T04:08:41,701 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-12T04:08:41,702 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-12T04:08:41,703 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T04:08:41,703 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-12T04:08:41,704 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-12T04:08:41,705 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T04:08:41,706 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T04:08:41,707 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-12T04:08:41,707 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/nfs.dump.dir in system properties and HBase conf 2024-11-12T04:08:41,708 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/java.io.tmpdir in system properties and HBase conf 2024-11-12T04:08:41,709 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T04:08:41,709 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-12T04:08:41,710 INFO [Time-limited test {}] hbase.HBaseTestingUtility(824): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-12T04:08:42,763 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-12T04:08:42,876 INFO [Time-limited test {}] log.Log(170): Logging initialized @3010ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-12T04:08:42,972 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T04:08:43,049 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T04:08:43,071 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T04:08:43,072 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T04:08:43,074 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T04:08:43,091 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T04:08:43,094 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@842d13f{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/hadoop.log.dir/,AVAILABLE} 2024-11-12T04:08:43,095 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@201ce359{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T04:08:43,295 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@2490b214{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/java.io.tmpdir/jetty-localhost-45153-hadoop-hdfs-3_4_1-tests_jar-_-any-16929936694685675786/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T04:08:43,305 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@1f7e2804{HTTP/1.1, (http/1.1)}{localhost:45153} 2024-11-12T04:08:43,305 INFO [Time-limited test {}] server.Server(415): Started @3440ms 2024-11-12T04:08:43,820 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T04:08:43,828 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T04:08:43,830 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T04:08:43,831 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T04:08:43,831 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T04:08:43,832 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@5b9056db{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/hadoop.log.dir/,AVAILABLE} 2024-11-12T04:08:43,832 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@50244d8f{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T04:08:43,931 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@19f1b88e{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/java.io.tmpdir/jetty-localhost-33505-hadoop-hdfs-3_4_1-tests_jar-_-any-5631750326367413246/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T04:08:43,932 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@2117487{HTTP/1.1, (http/1.1)}{localhost:33505} 2024-11-12T04:08:43,932 INFO [Time-limited test {}] server.Server(415): Started @4067ms 2024-11-12T04:08:43,979 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T04:08:44,747 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/cluster_b0a2b79c-cac7-9f39-3098-f01052b75b38/dfs/data/data1/current/BP-1580922183-172.17.0.2-1731384522326/current, will proceed with Du for space computation calculation, 2024-11-12T04:08:44,747 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/cluster_b0a2b79c-cac7-9f39-3098-f01052b75b38/dfs/data/data2/current/BP-1580922183-172.17.0.2-1731384522326/current, will proceed with Du for space computation calculation, 2024-11-12T04:08:44,780 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T04:08:44,834 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf0daf363ea17871c with lease ID 0x358fd793350e2ef: Processing first storage report for DS-ec3ff3ff-91c2-401f-ac91-ee43951db131 from datanode DatanodeRegistration(127.0.0.1:34483, datanodeUuid=c032ecd3-fe6c-4bfb-bf48-6308eca3ccbc, infoPort=41335, infoSecurePort=0, ipcPort=45791, storageInfo=lv=-57;cid=testClusterID;nsid=579396736;c=1731384522326) 2024-11-12T04:08:44,835 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf0daf363ea17871c with lease ID 0x358fd793350e2ef: from storage DS-ec3ff3ff-91c2-401f-ac91-ee43951db131 node DatanodeRegistration(127.0.0.1:34483, datanodeUuid=c032ecd3-fe6c-4bfb-bf48-6308eca3ccbc, infoPort=41335, infoSecurePort=0, ipcPort=45791, storageInfo=lv=-57;cid=testClusterID;nsid=579396736;c=1731384522326), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-12T04:08:44,835 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf0daf363ea17871c with lease ID 0x358fd793350e2ef: Processing first storage report for DS-18013ec5-18d5-4e23-a1b0-ac244563d072 from datanode DatanodeRegistration(127.0.0.1:34483, datanodeUuid=c032ecd3-fe6c-4bfb-bf48-6308eca3ccbc, infoPort=41335, infoSecurePort=0, ipcPort=45791, storageInfo=lv=-57;cid=testClusterID;nsid=579396736;c=1731384522326) 2024-11-12T04:08:44,836 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf0daf363ea17871c with lease ID 0x358fd793350e2ef: from storage DS-18013ec5-18d5-4e23-a1b0-ac244563d072 node DatanodeRegistration(127.0.0.1:34483, datanodeUuid=c032ecd3-fe6c-4bfb-bf48-6308eca3ccbc, infoPort=41335, infoSecurePort=0, ipcPort=45791, storageInfo=lv=-57;cid=testClusterID;nsid=579396736;c=1731384522326), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-12T04:08:44,871 DEBUG [Time-limited test {}] hbase.HBaseTestingUtility(703): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b 2024-11-12T04:08:44,954 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(259): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/cluster_b0a2b79c-cac7-9f39-3098-f01052b75b38/zookeeper_0, clientPort=50727, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/cluster_b0a2b79c-cac7-9f39-3098-f01052b75b38/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/cluster_b0a2b79c-cac7-9f39-3098-f01052b75b38/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-12T04:08:44,964 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(284): Started MiniZooKeeperCluster and ran 'stat' on client port=50727 2024-11-12T04:08:44,974 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T04:08:44,976 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T04:08:45,218 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741825_1001 (size=7) 2024-11-12T04:08:45,636 INFO [Time-limited test {}] util.FSUtils(490): Created version file at hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300 with version=8 2024-11-12T04:08:45,636 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1460): Setting hbase.fs.tmp.dir to hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/hbase-staging 2024-11-12T04:08:45,751 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-12T04:08:45,982 INFO [Time-limited test {}] client.ConnectionUtils(129): master/354fa2a6db59:0 server-side Connection retries=6 2024-11-12T04:08:45,998 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T04:08:45,998 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T04:08:45,999 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T04:08:45,999 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T04:08:45,999 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T04:08:46,112 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T04:08:46,163 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-12T04:08:46,171 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-12T04:08:46,175 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T04:08:46,196 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 28047 (auto-detected) 2024-11-12T04:08:46,197 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-12T04:08:46,214 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:36173 2024-11-12T04:08:46,221 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T04:08:46,222 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T04:08:46,233 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=master:36173 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:08:46,325 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:361730x0, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:08:46,327 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:36173-0x1012e84e2f80000 connected 2024-11-12T04:08:46,416 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T04:08:46,420 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T04:08:46,423 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T04:08:46,428 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=36173 2024-11-12T04:08:46,428 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=36173 2024-11-12T04:08:46,429 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=36173 2024-11-12T04:08:46,429 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=36173 2024-11-12T04:08:46,429 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=36173 2024-11-12T04:08:46,436 INFO [Time-limited test {}] master.HMaster(488): hbase.rootdir=hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300, hbase.cluster.distributed=false 2024-11-12T04:08:46,495 INFO [Time-limited test {}] client.ConnectionUtils(129): regionserver/354fa2a6db59:0 server-side Connection retries=6 2024-11-12T04:08:46,495 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T04:08:46,495 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T04:08:46,496 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T04:08:46,496 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T04:08:46,496 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T04:08:46,498 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T04:08:46,500 INFO [Time-limited test {}] ipc.NettyRpcServer(315): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T04:08:46,501 INFO [Time-limited test {}] ipc.NettyRpcServer(197): Bind to /172.17.0.2:40009 2024-11-12T04:08:46,503 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T04:08:46,513 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T04:08:46,514 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T04:08:46,517 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T04:08:46,520 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=regionserver:40009 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:08:46,531 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:400090x0, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:08:46,532 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:40009-0x1012e84e2f80001 connected 2024-11-12T04:08:46,532 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:400090x0, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T04:08:46,533 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T04:08:46,534 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T04:08:46,535 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40009 2024-11-12T04:08:46,535 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40009 2024-11-12T04:08:46,537 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40009 2024-11-12T04:08:46,539 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40009 2024-11-12T04:08:46,539 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40009 2024-11-12T04:08:46,541 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] master.HMaster(2445): Adding backup master ZNode /hbase/backup-masters/354fa2a6db59,36173,1731384525744 2024-11-12T04:08:46,555 DEBUG [M:0;354fa2a6db59:36173 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;354fa2a6db59:36173 2024-11-12T04:08:46,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T04:08:46,556 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T04:08:46,558 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/354fa2a6db59,36173,1731384525744 2024-11-12T04:08:46,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T04:08:46,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T04:08:46,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T04:08:46,586 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T04:08:46,587 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-12T04:08:46,588 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(111): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-12T04:08:46,588 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/354fa2a6db59,36173,1731384525744 from backup master directory 2024-11-12T04:08:46,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T04:08:46,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/354fa2a6db59,36173,1731384525744 2024-11-12T04:08:46,597 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T04:08:46,598 WARN [master/354fa2a6db59:0:becomeActiveMaster {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T04:08:46,598 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=354fa2a6db59,36173,1731384525744 2024-11-12T04:08:46,600 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-12T04:08:46,601 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-12T04:08:46,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741826_1002 (size=42) 2024-11-12T04:08:47,063 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] util.FSUtils(639): Created cluster ID file at hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/hbase.id with ID: f839b423-66d7-495f-b21c-29c55cacf7ac 2024-11-12T04:08:47,108 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T04:08:47,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T04:08:47,148 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T04:08:47,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741827_1003 (size=196) 2024-11-12T04:08:47,583 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] region.MasterRegion(372): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T04:08:47,586 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-12T04:08:47,603 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:147) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:160) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProvider(WALFactory.java:200) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:232) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:207) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:402) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:973) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2470) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:590) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.HMaster.lambda$run$1(HMaster.java:587) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:08:47,609 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T04:08:47,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741828_1004 (size=1189) 2024-11-12T04:08:48,063 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] regionserver.HRegion(7124): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/data/master/store 2024-11-12T04:08:48,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741829_1005 (size=34) 2024-11-12T04:08:48,484 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-12T04:08:48,485 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T04:08:48,486 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] regionserver.HRegion(1681): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T04:08:48,487 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] regionserver.HRegion(1703): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T04:08:48,487 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] regionserver.HRegion(1724): Waiting without time limit for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T04:08:48,487 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] regionserver.HRegion(1791): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T04:08:48,487 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] regionserver.HRegion(1801): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T04:08:48,487 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] regionserver.HRegion(1922): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T04:08:48,488 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] regionserver.HRegion(1635): Region close journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-12T04:08:48,490 WARN [master/354fa2a6db59:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/data/master/store/.initializing 2024-11-12T04:08:48,490 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/WALs/354fa2a6db59,36173,1731384525744 2024-11-12T04:08:48,497 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T04:08:48,507 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=354fa2a6db59%2C36173%2C1731384525744, suffix=, logDir=hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/WALs/354fa2a6db59,36173,1731384525744, archiveDir=hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/oldWALs, maxLogs=10 2024-11-12T04:08:48,525 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(599): When create output stream for /user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/WALs/354fa2a6db59,36173,1731384525744/354fa2a6db59%2C36173%2C1731384525744.1731384528511, exclude list is [], retry=0 2024-11-12T04:08:48,542 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-ec3ff3ff-91c2-401f-ac91-ee43951db131,DISK] 2024-11-12T04:08:48,545 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-12T04:08:48,577 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/WALs/354fa2a6db59,36173,1731384525744/354fa2a6db59%2C36173%2C1731384525744.1731384528511 2024-11-12T04:08:48,577 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41335:41335)] 2024-11-12T04:08:48,578 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] regionserver.HRegion(7285): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-12T04:08:48,578 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] regionserver.HRegion(894): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T04:08:48,581 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] regionserver.HRegion(7327): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T04:08:48,582 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] regionserver.HRegion(7330): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T04:08:48,615 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T04:08:48,641 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-12T04:08:48,645 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T04:08:48,648 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T04:08:48,649 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T04:08:48,653 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-12T04:08:48,653 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T04:08:48,654 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T04:08:48,655 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T04:08:48,658 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-12T04:08:48,658 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T04:08:48,659 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T04:08:48,659 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T04:08:48,662 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-12T04:08:48,662 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T04:08:48,663 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T04:08:48,667 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T04:08:48,668 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T04:08:48,678 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T04:08:48,683 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] regionserver.HRegion(1085): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T04:08:48,688 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T04:08:48,689 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] regionserver.HRegion(1102): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61481153, jitterRate=-0.08385942876338959}}}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T04:08:48,695 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] regionserver.HRegion(1001): Region open journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-12T04:08:48,696 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-12T04:08:48,724 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f1fdad7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:08:48,759 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] master.HMaster(882): No meta location available on zookeeper, skip migrating... 2024-11-12T04:08:48,769 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-12T04:08:48,770 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(633): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-12T04:08:48,772 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T04:08:48,773 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(653): Recovered RegionProcedureStore lease in 1 msec 2024-11-12T04:08:48,777 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(667): Loaded RegionProcedureStore in 4 msec 2024-11-12T04:08:48,778 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-12T04:08:48,800 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-12T04:08:48,814 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-12T04:08:48,822 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/balancer already deleted, retry=false 2024-11-12T04:08:48,826 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-12T04:08:48,827 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-12T04:08:48,836 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/normalizer already deleted, retry=false 2024-11-12T04:08:48,838 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-12T04:08:48,842 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-12T04:08:48,852 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/split already deleted, retry=false 2024-11-12T04:08:48,854 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-12T04:08:48,864 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/switch/merge already deleted, retry=false 2024-11-12T04:08:48,874 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-12T04:08:48,880 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(215): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-12T04:08:48,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T04:08:48,894 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T04:08:48,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T04:08:48,895 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T04:08:48,896 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] master.HMaster(826): Active/primary master=354fa2a6db59,36173,1731384525744, sessionid=0x1012e84e2f80000, setting cluster-up flag (Was=false) 2024-11-12T04:08:48,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T04:08:48,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T04:08:48,948 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-12T04:08:48,949 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=354fa2a6db59,36173,1731384525744 2024-11-12T04:08:48,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T04:08:48,969 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T04:08:49,022 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-12T04:08:49,024 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=354fa2a6db59,36173,1731384525744 2024-11-12T04:08:49,054 DEBUG [RS:0;354fa2a6db59:40009 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;354fa2a6db59:40009 2024-11-12T04:08:49,056 INFO [RS:0;354fa2a6db59:40009 {}] regionserver.HRegionServer(1008): ClusterId : f839b423-66d7-495f-b21c-29c55cacf7ac 2024-11-12T04:08:49,058 DEBUG [RS:0;354fa2a6db59:40009 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T04:08:49,071 DEBUG [RS:0;354fa2a6db59:40009 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T04:08:49,071 DEBUG [RS:0;354fa2a6db59:40009 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T04:08:49,082 DEBUG [RS:0;354fa2a6db59:40009 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T04:08:49,083 DEBUG [RS:0;354fa2a6db59:40009 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ff5efb3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:08:49,084 DEBUG [RS:0;354fa2a6db59:40009 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9ecfc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=354fa2a6db59/172.17.0.2:0 2024-11-12T04:08:49,087 INFO [RS:0;354fa2a6db59:40009 {}] regionserver.RegionServerCoprocessorHost(67): System coprocessor loading is enabled 2024-11-12T04:08:49,087 INFO [RS:0;354fa2a6db59:40009 {}] regionserver.RegionServerCoprocessorHost(68): Table coprocessor loading is enabled 2024-11-12T04:08:49,087 DEBUG [RS:0;354fa2a6db59:40009 {}] regionserver.HRegionServer(1090): About to register with Master. 2024-11-12T04:08:49,089 INFO [RS:0;354fa2a6db59:40009 {}] regionserver.HRegionServer(3073): reportForDuty to master=354fa2a6db59,36173,1731384525744 with isa=354fa2a6db59/172.17.0.2:40009, startcode=1731384526494 2024-11-12T04:08:49,100 DEBUG [RS:0;354fa2a6db59:40009 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T04:08:49,114 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT; InitMetaProcedure table=hbase:meta 2024-11-12T04:08:49,121 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(575): slop=0.2 2024-11-12T04:08:49,125 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(294): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-12T04:08:49,132 INFO [RS-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:40281, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T04:08:49,132 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] balancer.RegionLocationFinder(146): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 354fa2a6db59,36173,1731384525744 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-12T04:08:49,137 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/354fa2a6db59:0, corePoolSize=5, maxPoolSize=5 2024-11-12T04:08:49,138 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/354fa2a6db59:0, corePoolSize=5, maxPoolSize=5 2024-11-12T04:08:49,138 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/354fa2a6db59:0, corePoolSize=5, maxPoolSize=5 2024-11-12T04:08:49,138 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36173 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3280) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:593) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-12T04:08:49,138 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/354fa2a6db59:0, corePoolSize=5, maxPoolSize=5 2024-11-12T04:08:49,138 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/354fa2a6db59:0, corePoolSize=10, maxPoolSize=10 2024-11-12T04:08:49,138 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/354fa2a6db59:0, corePoolSize=1, maxPoolSize=1 2024-11-12T04:08:49,139 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/354fa2a6db59:0, corePoolSize=2, maxPoolSize=2 2024-11-12T04:08:49,139 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/354fa2a6db59:0, corePoolSize=1, maxPoolSize=1 2024-11-12T04:08:49,140 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731384559140 2024-11-12T04:08:49,142 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-12T04:08:49,143 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-12T04:08:49,144 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, locked=true; InitMetaProcedure table=hbase:meta 2024-11-12T04:08:49,145 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(75): BOOTSTRAP: creating hbase:meta region 2024-11-12T04:08:49,147 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-12T04:08:49,147 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-12T04:08:49,147 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-12T04:08:49,148 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-12T04:08:49,148 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T04:08:49,148 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T04:08:49,149 INFO [PEWorker-1 {}] util.FSTableDescriptors(133): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-12T04:08:49,150 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-12T04:08:49,152 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-12T04:08:49,152 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-12T04:08:49,154 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-12T04:08:49,155 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-12T04:08:49,162 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/354fa2a6db59:0:becomeActiveMaster-HFileCleaner.large.0-1731384529156,5,FailOnTimeoutGroup] 2024-11-12T04:08:49,163 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/354fa2a6db59:0:becomeActiveMaster-HFileCleaner.small.0-1731384529163,5,FailOnTimeoutGroup] 2024-11-12T04:08:49,163 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T04:08:49,164 DEBUG [RS:0;354fa2a6db59:40009 {}] regionserver.HRegionServer(3097): Master is not running yet 2024-11-12T04:08:49,164 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] master.HMaster(1680): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-12T04:08:49,164 WARN [RS:0;354fa2a6db59:40009 {}] regionserver.HRegionServer(1099): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-12T04:08:49,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741831_1007 (size=1039) 2024-11-12T04:08:49,165 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-12T04:08:49,165 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-12T04:08:49,265 INFO [RS:0;354fa2a6db59:40009 {}] regionserver.HRegionServer(3073): reportForDuty to master=354fa2a6db59,36173,1731384525744 with isa=354fa2a6db59/172.17.0.2:40009, startcode=1731384526494 2024-11-12T04:08:49,267 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36173 {}] master.ServerManager(332): Checking decommissioned status of RegionServer 354fa2a6db59,40009,1731384526494 2024-11-12T04:08:49,269 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36173 {}] master.ServerManager(486): Registering regionserver=354fa2a6db59,40009,1731384526494 2024-11-12T04:08:49,276 DEBUG [RS:0;354fa2a6db59:40009 {}] regionserver.HRegionServer(1725): Config from master: hbase.rootdir=hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300 2024-11-12T04:08:49,276 DEBUG [RS:0;354fa2a6db59:40009 {}] regionserver.HRegionServer(1725): Config from master: fs.defaultFS=hdfs://localhost:44407 2024-11-12T04:08:49,276 DEBUG [RS:0;354fa2a6db59:40009 {}] regionserver.HRegionServer(1725): Config from master: hbase.master.info.port=-1 2024-11-12T04:08:49,289 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T04:08:49,290 DEBUG [RS:0;354fa2a6db59:40009 {}] zookeeper.ZKUtil(111): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/354fa2a6db59,40009,1731384526494 2024-11-12T04:08:49,290 WARN [RS:0;354fa2a6db59:40009 {}] hbase.ZNodeClearer(69): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T04:08:49,290 INFO [RS:0;354fa2a6db59:40009 {}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T04:08:49,290 DEBUG [RS:0;354fa2a6db59:40009 {}] regionserver.HRegionServer(2100): logDir=hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/WALs/354fa2a6db59,40009,1731384526494 2024-11-12T04:08:49,292 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [354fa2a6db59,40009,1731384526494] 2024-11-12T04:08:49,302 DEBUG [RS:0;354fa2a6db59:40009 {}] regionserver.Replication(140): Replication stats-in-log period=300 seconds 2024-11-12T04:08:49,311 INFO [RS:0;354fa2a6db59:40009 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T04:08:49,325 INFO [RS:0;354fa2a6db59:40009 {}] regionserver.MemStoreFlusher(130): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T04:08:49,327 INFO [RS:0;354fa2a6db59:40009 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T04:08:49,328 INFO [RS:0;354fa2a6db59:40009 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T04:08:49,328 INFO [RS:0;354fa2a6db59:40009 {}] regionserver.HRegionServer$CompactionChecker(1988): CompactionChecker runs every PT1S 2024-11-12T04:08:49,333 INFO [RS:0;354fa2a6db59:40009 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T04:08:49,334 DEBUG [RS:0;354fa2a6db59:40009 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/354fa2a6db59:0, corePoolSize=1, maxPoolSize=1 2024-11-12T04:08:49,334 DEBUG [RS:0;354fa2a6db59:40009 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/354fa2a6db59:0, corePoolSize=1, maxPoolSize=1 2024-11-12T04:08:49,334 DEBUG [RS:0;354fa2a6db59:40009 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/354fa2a6db59:0, corePoolSize=1, maxPoolSize=1 2024-11-12T04:08:49,334 DEBUG [RS:0;354fa2a6db59:40009 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/354fa2a6db59:0, corePoolSize=1, maxPoolSize=1 2024-11-12T04:08:49,334 DEBUG [RS:0;354fa2a6db59:40009 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/354fa2a6db59:0, corePoolSize=1, maxPoolSize=1 2024-11-12T04:08:49,334 DEBUG [RS:0;354fa2a6db59:40009 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/354fa2a6db59:0, corePoolSize=2, maxPoolSize=2 2024-11-12T04:08:49,334 DEBUG [RS:0;354fa2a6db59:40009 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/354fa2a6db59:0, corePoolSize=1, maxPoolSize=1 2024-11-12T04:08:49,334 DEBUG [RS:0;354fa2a6db59:40009 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/354fa2a6db59:0, corePoolSize=1, maxPoolSize=1 2024-11-12T04:08:49,335 DEBUG [RS:0;354fa2a6db59:40009 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/354fa2a6db59:0, corePoolSize=1, maxPoolSize=1 2024-11-12T04:08:49,335 DEBUG [RS:0;354fa2a6db59:40009 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/354fa2a6db59:0, corePoolSize=1, maxPoolSize=1 2024-11-12T04:08:49,335 DEBUG [RS:0;354fa2a6db59:40009 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/354fa2a6db59:0, corePoolSize=1, maxPoolSize=1 2024-11-12T04:08:49,335 DEBUG [RS:0;354fa2a6db59:40009 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/354fa2a6db59:0, corePoolSize=3, maxPoolSize=3 2024-11-12T04:08:49,335 DEBUG [RS:0;354fa2a6db59:40009 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/354fa2a6db59:0, corePoolSize=3, maxPoolSize=3 2024-11-12T04:08:49,336 INFO [RS:0;354fa2a6db59:40009 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T04:08:49,336 INFO [RS:0;354fa2a6db59:40009 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T04:08:49,336 INFO [RS:0;354fa2a6db59:40009 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T04:08:49,336 INFO [RS:0;354fa2a6db59:40009 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T04:08:49,336 INFO [RS:0;354fa2a6db59:40009 {}] hbase.ChoreService(168): Chore ScheduledChore name=354fa2a6db59,40009,1731384526494-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T04:08:49,359 INFO [RS:0;354fa2a6db59:40009 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T04:08:49,361 INFO [RS:0;354fa2a6db59:40009 {}] hbase.ChoreService(168): Chore ScheduledChore name=354fa2a6db59,40009,1731384526494-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T04:08:49,378 INFO [RS:0;354fa2a6db59:40009 {}] regionserver.Replication(204): 354fa2a6db59,40009,1731384526494 started 2024-11-12T04:08:49,378 INFO [RS:0;354fa2a6db59:40009 {}] regionserver.HRegionServer(1767): Serving as 354fa2a6db59,40009,1731384526494, RpcServer on 354fa2a6db59/172.17.0.2:40009, sessionid=0x1012e84e2f80001 2024-11-12T04:08:49,379 DEBUG [RS:0;354fa2a6db59:40009 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T04:08:49,379 DEBUG [RS:0;354fa2a6db59:40009 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 354fa2a6db59,40009,1731384526494 2024-11-12T04:08:49,379 DEBUG [RS:0;354fa2a6db59:40009 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '354fa2a6db59,40009,1731384526494' 2024-11-12T04:08:49,379 DEBUG [RS:0;354fa2a6db59:40009 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T04:08:49,380 DEBUG [RS:0;354fa2a6db59:40009 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T04:08:49,381 DEBUG [RS:0;354fa2a6db59:40009 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T04:08:49,381 DEBUG [RS:0;354fa2a6db59:40009 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T04:08:49,381 DEBUG [RS:0;354fa2a6db59:40009 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 354fa2a6db59,40009,1731384526494 2024-11-12T04:08:49,381 DEBUG [RS:0;354fa2a6db59:40009 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '354fa2a6db59,40009,1731384526494' 2024-11-12T04:08:49,381 DEBUG [RS:0;354fa2a6db59:40009 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T04:08:49,382 DEBUG [RS:0;354fa2a6db59:40009 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T04:08:49,382 DEBUG [RS:0;354fa2a6db59:40009 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T04:08:49,383 INFO [RS:0;354fa2a6db59:40009 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T04:08:49,383 INFO [RS:0;354fa2a6db59:40009 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T04:08:49,490 INFO [RS:0;354fa2a6db59:40009 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T04:08:49,494 INFO [RS:0;354fa2a6db59:40009 {}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=354fa2a6db59%2C40009%2C1731384526494, suffix=, logDir=hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/WALs/354fa2a6db59,40009,1731384526494, archiveDir=hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/oldWALs, maxLogs=32 2024-11-12T04:08:49,508 DEBUG [RS:0;354fa2a6db59:40009 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(599): When create output stream for /user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/WALs/354fa2a6db59,40009,1731384526494/354fa2a6db59%2C40009%2C1731384526494.1731384529497, exclude list is [], retry=0 2024-11-12T04:08:49,513 DEBUG [RS-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-ec3ff3ff-91c2-401f-ac91-ee43951db131,DISK] 2024-11-12T04:08:49,516 INFO [RS:0;354fa2a6db59:40009 {}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/WALs/354fa2a6db59,40009,1731384526494/354fa2a6db59%2C40009%2C1731384526494.1731384529497 2024-11-12T04:08:49,516 DEBUG [RS:0;354fa2a6db59:40009 {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41335:41335)] 2024-11-12T04:08:49,567 INFO [PEWorker-1 {}] util.FSTableDescriptors(140): Updated hbase:meta table descriptor to hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1039 2024-11-12T04:08:49,568 INFO [PEWorker-1 {}] regionserver.HRegion(7106): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300 2024-11-12T04:08:49,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741833_1009 (size=32) 2024-11-12T04:08:49,984 DEBUG [PEWorker-1 {}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T04:08:49,988 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T04:08:49,992 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T04:08:49,993 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T04:08:49,994 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T04:08:49,995 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T04:08:49,999 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T04:08:49,999 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T04:08:50,000 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T04:08:50,001 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T04:08:50,003 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T04:08:50,003 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T04:08:50,004 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T04:08:50,006 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/data/hbase/meta/1588230740 2024-11-12T04:08:50,007 DEBUG [PEWorker-1 {}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/data/hbase/meta/1588230740 2024-11-12T04:08:50,009 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T04:08:50,012 DEBUG [PEWorker-1 {}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-12T04:08:50,016 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T04:08:50,017 INFO [PEWorker-1 {}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60493588, jitterRate=-0.09857529401779175}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T04:08:50,020 DEBUG [PEWorker-1 {}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-12T04:08:50,020 DEBUG [PEWorker-1 {}] regionserver.HRegion(1681): Closing 1588230740, disabling compactions & flushes 2024-11-12T04:08:50,020 INFO [PEWorker-1 {}] regionserver.HRegion(1703): Closing region hbase:meta,,1.1588230740 2024-11-12T04:08:50,020 DEBUG [PEWorker-1 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:meta,,1.1588230740 2024-11-12T04:08:50,020 DEBUG [PEWorker-1 {}] regionserver.HRegion(1791): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T04:08:50,020 DEBUG [PEWorker-1 {}] regionserver.HRegion(1801): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T04:08:50,022 INFO [PEWorker-1 {}] regionserver.HRegion(1922): Closed hbase:meta,,1.1588230740 2024-11-12T04:08:50,022 DEBUG [PEWorker-1 {}] regionserver.HRegion(1635): Region close journal for 1588230740: 2024-11-12T04:08:50,024 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(96): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, locked=true; InitMetaProcedure table=hbase:meta 2024-11-12T04:08:50,024 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(107): Going to assign meta 2024-11-12T04:08:50,029 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T04:08:50,036 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T04:08:50,038 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(264): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-12T04:08:50,191 DEBUG [354fa2a6db59:36173 {}] assignment.AssignmentManager(2444): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-12T04:08:50,200 INFO [PEWorker-3 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=354fa2a6db59,40009,1731384526494 2024-11-12T04:08:50,206 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 354fa2a6db59,40009,1731384526494, state=OPENING 2024-11-12T04:08:50,219 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-12T04:08:50,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T04:08:50,227 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T04:08:50,229 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T04:08:50,230 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T04:08:50,232 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE; OpenRegionProcedure 1588230740, server=354fa2a6db59,40009,1731384526494}] 2024-11-12T04:08:50,408 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:08:50,409 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T04:08:50,413 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34648, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T04:08:50,423 INFO [RS_OPEN_META-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(135): Open hbase:meta,,1.1588230740 2024-11-12T04:08:50,423 INFO [RS_OPEN_META-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(183): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T04:08:50,424 INFO [RS_OPEN_META-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T04:08:50,427 INFO [RS_OPEN_META-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(500): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=354fa2a6db59%2C40009%2C1731384526494.meta, suffix=.meta, logDir=hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/WALs/354fa2a6db59,40009,1731384526494, archiveDir=hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/oldWALs, maxLogs=32 2024-11-12T04:08:50,440 DEBUG [RS_OPEN_META-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(599): When create output stream for /user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/WALs/354fa2a6db59,40009,1731384526494/354fa2a6db59%2C40009%2C1731384526494.meta.1731384530428.meta, exclude list is [], retry=0 2024-11-12T04:08:50,444 DEBUG [RS-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-ec3ff3ff-91c2-401f-ac91-ee43951db131,DISK] 2024-11-12T04:08:50,447 INFO [RS_OPEN_META-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(841): New WAL /user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/WALs/354fa2a6db59,40009,1731384526494/354fa2a6db59%2C40009%2C1731384526494.meta.1731384530428.meta 2024-11-12T04:08:50,447 DEBUG [RS_OPEN_META-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41335:41335)] 2024-11-12T04:08:50,448 DEBUG [RS_OPEN_META-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7285): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T04:08:50,449 DEBUG [RS_OPEN_META-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T04:08:50,507 DEBUG [RS_OPEN_META-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7999): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T04:08:50,511 INFO [RS_OPEN_META-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(436): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T04:08:50,515 DEBUG [RS_OPEN_META-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T04:08:50,515 DEBUG [RS_OPEN_META-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(894): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T04:08:50,516 DEBUG [RS_OPEN_META-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7327): checking encryption for 1588230740 2024-11-12T04:08:50,516 DEBUG [RS_OPEN_META-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7330): checking classloading for 1588230740 2024-11-12T04:08:50,519 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T04:08:50,521 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T04:08:50,521 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T04:08:50,522 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T04:08:50,522 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T04:08:50,524 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T04:08:50,524 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T04:08:50,525 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T04:08:50,525 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T04:08:50,526 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T04:08:50,527 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T04:08:50,528 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T04:08:50,530 DEBUG [RS_OPEN_META-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/data/hbase/meta/1588230740 2024-11-12T04:08:50,533 DEBUG [RS_OPEN_META-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/data/hbase/meta/1588230740 2024-11-12T04:08:50,536 DEBUG [RS_OPEN_META-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (42.7 M)) instead. 2024-11-12T04:08:50,539 DEBUG [RS_OPEN_META-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1085): writing seq id for 1588230740 2024-11-12T04:08:50,542 INFO [RS_OPEN_META-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1102): Opened 1588230740; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58782009, jitterRate=-0.12407980859279633}}}, FlushLargeStoresPolicy{flushSizeLowerBound=44739242} 2024-11-12T04:08:50,544 DEBUG [RS_OPEN_META-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1001): Region open journal for 1588230740: 2024-11-12T04:08:50,552 INFO [RS_OPEN_META-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731384530403 2024-11-12T04:08:50,563 DEBUG [RS_OPEN_META-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T04:08:50,563 INFO [RS_OPEN_META-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(164): Opened hbase:meta,,1.1588230740 2024-11-12T04:08:50,564 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=354fa2a6db59,40009,1731384526494 2024-11-12T04:08:50,566 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 354fa2a6db59,40009,1731384526494, state=OPEN 2024-11-12T04:08:50,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T04:08:50,601 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T04:08:50,601 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T04:08:50,601 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T04:08:50,610 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=3, resume processing ppid=2 2024-11-12T04:08:50,611 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=3, ppid=2, state=SUCCESS; OpenRegionProcedure 1588230740, server=354fa2a6db59,40009,1731384526494 in 369 msec 2024-11-12T04:08:50,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=2, resume processing ppid=1 2024-11-12T04:08:50,620 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=2, ppid=1, state=SUCCESS; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 583 msec 2024-11-12T04:08:50,625 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=1, state=SUCCESS; InitMetaProcedure table=hbase:meta in 1.5540 sec 2024-11-12T04:08:50,625 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] master.HMaster(1088): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731384530625, completionTime=-1 2024-11-12T04:08:50,625 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] master.ServerManager(907): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-12T04:08:50,625 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] assignment.AssignmentManager(1747): Joining cluster... 2024-11-12T04:08:50,660 DEBUG [hconnection-0x989010-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T04:08:50,662 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34650, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T04:08:50,677 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] assignment.AssignmentManager(1759): Number of RegionServers=1 2024-11-12T04:08:50,677 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731384590677 2024-11-12T04:08:50,677 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(81): ADDED pid=-1, state=WAITING_TIMEOUT; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731384650677 2024-11-12T04:08:50,677 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] assignment.AssignmentManager(1766): Joined the cluster in 51 msec 2024-11-12T04:08:50,713 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=354fa2a6db59,36173,1731384525744-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T04:08:50,713 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=354fa2a6db59,36173,1731384525744-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T04:08:50,713 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=354fa2a6db59,36173,1731384525744-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T04:08:50,714 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-354fa2a6db59:36173, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T04:08:50,715 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-12T04:08:50,720 DEBUG [master/354fa2a6db59:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-11-12T04:08:50,723 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] master.TableNamespaceManager(92): Namespace table not found. Creating... 2024-11-12T04:08:50,724 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] master.HMaster(2425): Client=null/null create 'hbase:namespace', {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-12T04:08:50,730 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION; CreateTableProcedure table=hbase:namespace 2024-11-12T04:08:50,734 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T04:08:50,735 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T04:08:50,738 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T04:08:50,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741835_1011 (size=358) 2024-11-12T04:08:51,155 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(7106): creating {ENCODED => ca1f0981bae46f8dd99b7b3bc2a86bb7, NAME => 'hbase:namespace,,1731384530724.ca1f0981bae46f8dd99b7b3bc2a86bb7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:namespace', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '10', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300 2024-11-12T04:08:51,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741836_1012 (size=42) 2024-11-12T04:08:51,567 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(894): Instantiated hbase:namespace,,1731384530724.ca1f0981bae46f8dd99b7b3bc2a86bb7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T04:08:51,568 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1681): Closing ca1f0981bae46f8dd99b7b3bc2a86bb7, disabling compactions & flushes 2024-11-12T04:08:51,568 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1703): Closing region hbase:namespace,,1731384530724.ca1f0981bae46f8dd99b7b3bc2a86bb7. 2024-11-12T04:08:51,568 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1724): Waiting without time limit for close lock on hbase:namespace,,1731384530724.ca1f0981bae46f8dd99b7b3bc2a86bb7. 2024-11-12T04:08:51,568 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1791): Acquired close lock on hbase:namespace,,1731384530724.ca1f0981bae46f8dd99b7b3bc2a86bb7. after waiting 0 ms 2024-11-12T04:08:51,568 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1801): Updates disabled for region hbase:namespace,,1731384530724.ca1f0981bae46f8dd99b7b3bc2a86bb7. 2024-11-12T04:08:51,568 INFO [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1922): Closed hbase:namespace,,1731384530724.ca1f0981bae46f8dd99b7b3bc2a86bb7. 2024-11-12T04:08:51,568 DEBUG [RegionOpenAndInit-hbase:namespace-pool-0 {}] regionserver.HRegion(1635): Region close journal for ca1f0981bae46f8dd99b7b3bc2a86bb7: 2024-11-12T04:08:51,572 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T04:08:51,583 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":2,"row":"hbase:namespace,,1731384530724.ca1f0981bae46f8dd99b7b3bc2a86bb7.","families":{"info":[{"qualifier":"regioninfo","vlen":41,"tag":[],"timestamp":"1731384531574"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731384531574"}]},"ts":"1731384531574"} 2024-11-12T04:08:51,604 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1516): Added 1 regions to meta. 2024-11-12T04:08:51,606 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T04:08:51,609 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731384531606"}]},"ts":"1731384531606"} 2024-11-12T04:08:51,613 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLING in hbase:meta 2024-11-12T04:08:51,649 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=ca1f0981bae46f8dd99b7b3bc2a86bb7, ASSIGN}] 2024-11-12T04:08:51,653 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(786): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE; TransitRegionStateProcedure table=hbase:namespace, region=ca1f0981bae46f8dd99b7b3bc2a86bb7, ASSIGN 2024-11-12T04:08:51,655 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(264): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, locked=true; TransitRegionStateProcedure table=hbase:namespace, region=ca1f0981bae46f8dd99b7b3bc2a86bb7, ASSIGN; state=OFFLINE, location=354fa2a6db59,40009,1731384526494; forceNewPlan=false, retain=false 2024-11-12T04:08:51,807 INFO [PEWorker-5 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=ca1f0981bae46f8dd99b7b3bc2a86bb7, regionState=OPENING, regionLocation=354fa2a6db59,40009,1731384526494 2024-11-12T04:08:51,815 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE; OpenRegionProcedure ca1f0981bae46f8dd99b7b3bc2a86bb7, server=354fa2a6db59,40009,1731384526494}] 2024-11-12T04:08:51,973 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:08:51,984 INFO [RS_OPEN_PRIORITY_REGION-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(135): Open hbase:namespace,,1731384530724.ca1f0981bae46f8dd99b7b3bc2a86bb7. 2024-11-12T04:08:51,984 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7285): Opening region: {ENCODED => ca1f0981bae46f8dd99b7b3bc2a86bb7, NAME => 'hbase:namespace,,1731384530724.ca1f0981bae46f8dd99b7b3bc2a86bb7.', STARTKEY => '', ENDKEY => ''} 2024-11-12T04:08:51,985 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table namespace ca1f0981bae46f8dd99b7b3bc2a86bb7 2024-11-12T04:08:51,985 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(894): Instantiated hbase:namespace,,1731384530724.ca1f0981bae46f8dd99b7b3bc2a86bb7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T04:08:51,985 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7327): checking encryption for ca1f0981bae46f8dd99b7b3bc2a86bb7 2024-11-12T04:08:51,985 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(7330): checking classloading for ca1f0981bae46f8dd99b7b3bc2a86bb7 2024-11-12T04:08:51,987 INFO [StoreOpener-ca1f0981bae46f8dd99b7b3bc2a86bb7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region ca1f0981bae46f8dd99b7b3bc2a86bb7 2024-11-12T04:08:51,990 INFO [StoreOpener-ca1f0981bae46f8dd99b7b3bc2a86bb7-1 {}] compactions.CompactionConfiguration(181): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region ca1f0981bae46f8dd99b7b3bc2a86bb7 columnFamilyName info 2024-11-12T04:08:51,990 DEBUG [StoreOpener-ca1f0981bae46f8dd99b7b3bc2a86bb7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T04:08:51,991 INFO [StoreOpener-ca1f0981bae46f8dd99b7b3bc2a86bb7-1 {}] regionserver.HStore(327): Store=ca1f0981bae46f8dd99b7b3bc2a86bb7/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T04:08:51,993 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/data/hbase/namespace/ca1f0981bae46f8dd99b7b3bc2a86bb7 2024-11-12T04:08:51,994 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(5301): Found 0 recovered edits file(s) under hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/data/hbase/namespace/ca1f0981bae46f8dd99b7b3bc2a86bb7 2024-11-12T04:08:51,999 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1085): writing seq id for ca1f0981bae46f8dd99b7b3bc2a86bb7 2024-11-12T04:08:52,003 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] wal.WALSplitUtil(409): Wrote file=hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/data/hbase/namespace/ca1f0981bae46f8dd99b7b3bc2a86bb7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T04:08:52,004 INFO [RS_OPEN_PRIORITY_REGION-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1102): Opened ca1f0981bae46f8dd99b7b3bc2a86bb7; next sequenceid=2; SteppingSplitPolicysuper{IncreasingToUpperBoundRegionSplitPolicy{initialSize=268435456, ConstantSizeRegionSplitPolicy{desiredMaxFileSize=65044064, jitterRate=-0.03076791763305664}}}, FlushLargeStoresPolicy{flushSizeLowerBound=-1} 2024-11-12T04:08:52,006 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegion(1001): Region open journal for ca1f0981bae46f8dd99b7b3bc2a86bb7: 2024-11-12T04:08:52,008 INFO [RS_OPEN_PRIORITY_REGION-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2601): Post open deploy tasks for hbase:namespace,,1731384530724.ca1f0981bae46f8dd99b7b3bc2a86bb7., pid=6, masterSystemTime=1731384531973 2024-11-12T04:08:52,011 DEBUG [RS_OPEN_PRIORITY_REGION-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] regionserver.HRegionServer(2628): Finished post open deploy task for hbase:namespace,,1731384530724.ca1f0981bae46f8dd99b7b3bc2a86bb7. 2024-11-12T04:08:52,011 INFO [RS_OPEN_PRIORITY_REGION-regionserver/354fa2a6db59:0-0 {event_type=M_RS_OPEN_PRIORITY_REGION, pid=6}] handler.AssignRegionHandler(164): Opened hbase:namespace,,1731384530724.ca1f0981bae46f8dd99b7b3bc2a86bb7. 2024-11-12T04:08:52,012 INFO [PEWorker-2 {}] assignment.RegionStateStore(202): pid=5 updating hbase:meta row=ca1f0981bae46f8dd99b7b3bc2a86bb7, regionState=OPEN, openSeqNum=2, regionLocation=354fa2a6db59,40009,1731384526494 2024-11-12T04:08:52,020 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=6, resume processing ppid=5 2024-11-12T04:08:52,021 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=6, ppid=5, state=SUCCESS; OpenRegionProcedure ca1f0981bae46f8dd99b7b3bc2a86bb7, server=354fa2a6db59,40009,1731384526494 in 201 msec 2024-11-12T04:08:52,024 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=5, resume processing ppid=4 2024-11-12T04:08:52,024 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=5, ppid=4, state=SUCCESS; TransitRegionStateProcedure table=hbase:namespace, region=ca1f0981bae46f8dd99b7b3bc2a86bb7, ASSIGN in 371 msec 2024-11-12T04:08:52,025 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T04:08:52,025 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(2113): Put {"totalColumns":1,"row":"hbase:namespace","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731384532025"}]},"ts":"1731384532025"} 2024-11-12T04:08:52,028 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(1655): Updated tableName=hbase:namespace, state=ENABLED in hbase:meta 2024-11-12T04:08:52,040 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] zookeeper.ZKUtil(113): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/namespace 2024-11-12T04:08:52,041 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, locked=true; CreateTableProcedure table=hbase:namespace execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T04:08:52,044 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=4, state=SUCCESS; CreateTableProcedure table=hbase:namespace in 1.3160 sec 2024-11-12T04:08:52,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/namespace 2024-11-12T04:08:52,047 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T04:08:52,048 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T04:08:52,075 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=7, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=default 2024-11-12T04:08:52,097 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-12T04:08:52,112 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=7, state=SUCCESS; CreateNamespaceProcedure, namespace=default in 39 msec 2024-11-12T04:08:52,121 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1098): Stored pid=8, state=RUNNABLE:CREATE_NAMESPACE_PREPARE; CreateNamespaceProcedure, namespace=hbase 2024-11-12T04:08:52,144 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/namespace 2024-11-12T04:08:52,157 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=8, state=SUCCESS; CreateNamespaceProcedure, namespace=hbase in 35 msec 2024-11-12T04:08:52,186 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/default 2024-11-12T04:08:52,203 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/namespace/hbase 2024-11-12T04:08:52,203 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] master.HMaster(1218): Master has completed initialization 5.605sec 2024-11-12T04:08:52,207 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-12T04:08:52,210 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-12T04:08:52,212 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-12T04:08:52,213 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-12T04:08:52,214 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-12T04:08:52,216 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=354fa2a6db59,36173,1731384525744-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T04:08:52,217 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=354fa2a6db59,36173,1731384525744-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-12T04:08:52,240 DEBUG [master/354fa2a6db59:0:becomeActiveMaster {}] master.HMaster(1321): Balancer post startup initialization complete, took 0 seconds 2024-11-12T04:08:52,241 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-12T04:08:52,242 INFO [master/354fa2a6db59:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=354fa2a6db59,36173,1731384525744-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T04:08:52,262 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x53d9c032 to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7c2a5090 2024-11-12T04:08:52,263 WARN [Time-limited test {}] client.ZKConnectionRegistry(90): ZKConnectionRegistry is deprecated. See https://hbase.apache.org/book.html#client.rpcconnectionregistry 2024-11-12T04:08:52,275 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47ab1946, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:08:52,279 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-12T04:08:52,279 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-12T04:08:52,287 DEBUG [hconnection-0x485c112f-shared-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T04:08:52,294 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34660, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T04:08:52,302 INFO [Time-limited test {}] hbase.HBaseTestingUtility(1199): Minicluster is up; activeMaster=354fa2a6db59,36173,1731384525744 2024-11-12T04:08:52,304 DEBUG [Time-limited test {}] zookeeper.ReadOnlyZKClient(149): Connect 0x502fe4e9 to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@74e5f234 2024-11-12T04:08:52,316 DEBUG [ReadOnlyZKClient-127.0.0.1:50727@0x502fe4e9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d81f331, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:08:52,332 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[0] Thread=222, OpenFileDescriptor=448, MaxFileDescriptor=1048576, SystemLoadAverage=348, ProcessCount=11, AvailableMemoryMB=2620 2024-11-12T04:08:52,361 DEBUG [Time-limited test {}] client.ConnectionUtils(573): Start fetching master stub from registry 2024-11-12T04:08:52,362 DEBUG [ReadOnlyZKClient-127.0.0.1:50727@0x502fe4e9 {}] client.AsyncConnectionImpl(310): The fetched master address is 354fa2a6db59,36173,1731384525744 2024-11-12T04:08:52,364 DEBUG [ReadOnlyZKClient-127.0.0.1:50727@0x502fe4e9 {}] client.ConnectionUtils(581): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7d9829c7 2024-11-12T04:08:52,365 DEBUG [ReadOnlyZKClient-127.0.0.1:50727@0x502fe4e9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T04:08:52,368 INFO [RS-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43442, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T04:08:52,370 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-12T04:08:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=9, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-12T04:08:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-12T04:08:52,397 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:08:52,398 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:08:52,406 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:08:52,406 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80004 connected 2024-11-12T04:08:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-12T04:08:52,515 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80004, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:08:52,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T04:08:52,536 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T04:08:52,559 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:08:52,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-12T04:08:52,726 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:08:52,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=10 2024-11-12T04:08:52,728 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-12T04:08:52,750 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=10}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:08:52,768 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:08:52,833 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-12T04:08:52,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=10 2024-11-12T04:08:52,841 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:08:52,845 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=10, resume processing ppid=9 2024-11-12T04:08:52,845 INFO [PEWorker-5 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-12T04:08:52,845 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=10, ppid=9, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 282 msec 2024-11-12T04:08:52,848 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=9, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 474 msec 2024-11-12T04:08:52,868 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6d4c8153 to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7a0868b8 2024-11-12T04:08:52,878 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:08:52,879 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:08:52,879 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@241e22e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:08:52,882 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:08:52,894 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:08:52,895 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:08:52,895 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f80006 connected 2024-11-12T04:08:53,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=9 2024-11-12T04:08:53,009 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-12T04:08:53,012 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-12T04:08:53,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=11, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:08:53,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=11 2024-11-12T04:08:53,034 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:08:53,035 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:08:53,044 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:08:53,045 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80007 connected 2024-11-12T04:08:53,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=11 2024-11-12T04:08:53,153 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80007, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:08:53,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:08:53,324 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:08:53,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=12 2024-11-12T04:08:53,326 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:08:53,331 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-12T04:08:53,331 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-12T04:08:53,332 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:08:53,332 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f80006, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:08:53,333 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f80006, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:08:53,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=11 2024-11-12T04:08:53,444 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f80006, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:08:53,445 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=12}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6d4c8153 to 127.0.0.1:50727 2024-11-12T04:08:53,445 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=12}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:08:53,462 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:08:53,463 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-12T04:08:53,463 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-12T04:08:53,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=12 2024-11-12T04:08:53,466 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:08:53,469 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=12, resume processing ppid=11 2024-11-12T04:08:53,469 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-12T04:08:53,469 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=12, ppid=11, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 299 msec 2024-11-12T04:08:53,472 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=11, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 456 msec 2024-11-12T04:08:53,492 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x515280ac to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5e6469b8 2024-11-12T04:08:53,503 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:08:53,503 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:08:53,504 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@43d1f30, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:08:53,505 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:08:53,514 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:08:53,514 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:08:53,514 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f80009 connected 2024-11-12T04:08:53,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=11 2024-11-12T04:08:53,650 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:08:53,661 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:08:53,663 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:08:53,665 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:08:53,668 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-11-12T04:08:53,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=13, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:08:53,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=13 2024-11-12T04:08:53,686 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:08:53,687 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:08:53,697 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:08:53,698 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8000a connected 2024-11-12T04:08:53,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=13 2024-11-12T04:08:53,806 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8000a, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:08:53,848 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:08:53,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=13 2024-11-12T04:08:54,002 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:08:54,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=14 2024-11-12T04:08:54,003 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:08:54,005 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-12T04:08:54,005 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-12T04:08:54,005 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:08:54,005 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f80009, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:08:54,006 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f80009, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:08:54,119 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f80009, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:08:54,119 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=14}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x515280ac to 127.0.0.1:50727 2024-11-12T04:08:54,120 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=14}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:08:54,141 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:08:54,142 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-12T04:08:54,142 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-12T04:08:54,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=14 2024-11-12T04:08:54,145 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:08:54,149 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=14, resume processing ppid=13 2024-11-12T04:08:54,149 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=14, ppid=13, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 297 msec 2024-11-12T04:08:54,149 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-11-12T04:08:54,152 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=13, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 481 msec 2024-11-12T04:08:54,179 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x630fd729 to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@652336ff 2024-11-12T04:08:54,190 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:08:54,190 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:08:54,190 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3615875e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:08:54,192 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:08:54,202 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:08:54,203 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:08:54,203 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f8000c connected 2024-11-12T04:08:54,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=13 2024-11-12T04:08:54,298 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:08:54,299 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:08:54,301 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:08:54,302 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:08:54,304 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:08:54,305 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-11-12T04:08:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=15, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:08:54,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-12T04:08:54,328 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:08:54,330 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:08:54,339 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:08:54,339 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8000d connected 2024-11-12T04:08:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-12T04:08:54,453 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8000d, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:08:54,470 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:08:54,623 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:08:54,624 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=16 2024-11-12T04:08:54,624 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:08:54,625 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-12T04:08:54,625 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-12T04:08:54,625 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:08:54,625 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f8000c, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:08:54,626 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f8000c, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:08:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-12T04:08:54,736 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f8000c, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:08:54,736 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=16}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x630fd729 to 127.0.0.1:50727 2024-11-12T04:08:54,736 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=16}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:08:54,752 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:08:54,752 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-12T04:08:54,752 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-12T04:08:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=16 2024-11-12T04:08:54,756 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:08:54,761 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=16, resume processing ppid=15 2024-11-12T04:08:54,761 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-11-12T04:08:54,761 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=16, ppid=15, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 286 msec 2024-11-12T04:08:54,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=15, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 455 msec 2024-11-12T04:08:54,782 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5047598c to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@6a737b63 2024-11-12T04:08:54,794 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:08:54,795 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:08:54,795 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@2239bb57, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:08:54,796 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:08:54,806 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:08:54,806 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:08:54,806 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f8000f connected 2024-11-12T04:08:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=15 2024-11-12T04:08:54,939 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:08:54,939 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:08:54,941 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:08:54,943 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:08:54,945 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-12T04:08:54,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=17, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:08:54,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=17 2024-11-12T04:08:54,967 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:08:54,968 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:08:54,977 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:08:54,977 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80010 connected 2024-11-12T04:08:55,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=17 2024-11-12T04:08:55,086 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80010, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:08:55,102 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:08:55,256 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:08:55,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=18 2024-11-12T04:08:55,258 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:08:55,261 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-12T04:08:55,261 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-12T04:08:55,261 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:08:55,262 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f8000f, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:08:55,263 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f8000f, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:08:55,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=17 2024-11-12T04:08:55,307 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-12T04:08:55,309 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:namespace' 2024-11-12T04:08:55,373 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f8000f, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:08:55,373 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=18}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5047598c to 127.0.0.1:50727 2024-11-12T04:08:55,373 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=18}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:08:55,394 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:08:55,394 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-12T04:08:55,394 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-12T04:08:55,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=18 2024-11-12T04:08:55,398 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:08:55,402 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=18, resume processing ppid=17 2024-11-12T04:08:55,402 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-12T04:08:55,402 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=18, ppid=17, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 296 msec 2024-11-12T04:08:55,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=17, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 457 msec 2024-11-12T04:08:55,424 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4614fc3e to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1b5ee1c5 2024-11-12T04:08:55,436 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:08:55,437 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:08:55,437 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@259e460d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:08:55,438 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:08:55,448 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:08:55,448 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:08:55,448 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f80012 connected 2024-11-12T04:08:55,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=17 2024-11-12T04:08:55,580 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:08:55,581 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:08:55,583 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:08:55,584 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-11-12T04:08:55,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=19, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:08:55,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=19 2024-11-12T04:08:55,601 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:08:55,602 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:08:55,610 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:08:55,611 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80013 connected 2024-11-12T04:08:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=19 2024-11-12T04:08:55,719 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80013, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:08:55,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:08:55,888 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:08:55,888 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=20 2024-11-12T04:08:55,889 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:08:55,891 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-12T04:08:55,891 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-12T04:08:55,891 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:08:55,891 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f80012, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:08:55,892 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f80012, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:08:55,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=19 2024-11-12T04:08:56,006 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f80012, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:08:56,007 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=20}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4614fc3e to 127.0.0.1:50727 2024-11-12T04:08:56,007 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=20}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:08:56,026 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T04:08:56,030 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:08:56,031 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-12T04:08:56,031 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-12T04:08:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=20 2024-11-12T04:08:56,035 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:08:56,038 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=20, resume processing ppid=19 2024-11-12T04:08:56,038 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-11-12T04:08:56,038 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=20, ppid=19, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 302 msec 2024-11-12T04:08:56,041 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=19, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 454 msec 2024-11-12T04:08:56,060 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x594796f0 to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@87bbfe4 2024-11-12T04:08:56,069 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:08:56,070 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:08:56,070 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3f7ce0d8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:08:56,071 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:08:56,081 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:08:56,081 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:08:56,081 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f80015 connected 2024-11-12T04:08:56,160 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-12T04:08:56,161 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-12T04:08:56,167 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_namespace 2024-11-12T04:08:56,167 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_namespace Metrics about Tables on a single HBase RegionServer 2024-11-12T04:08:56,170 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T04:08:56,170 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-12T04:08:56,171 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-12T04:08:56,171 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-12T04:08:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=19 2024-11-12T04:08:56,219 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:08:56,219 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:08:56,221 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-12T04:08:56,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=21, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:08:56,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-12T04:08:56,243 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:08:56,244 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:08:56,252 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:08:56,252 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80016 connected 2024-11-12T04:08:56,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-12T04:08:56,361 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80016, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:08:56,374 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:08:56,528 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:08:56,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=22 2024-11-12T04:08:56,530 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:08:56,534 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-12T04:08:56,535 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-12T04:08:56,535 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:08:56,535 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f80015, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:08:56,536 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f80015, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:08:56,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-12T04:08:56,648 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f80015, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:08:56,648 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=22}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x594796f0 to 127.0.0.1:50727 2024-11-12T04:08:56,648 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=22}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:08:56,668 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:08:56,668 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-12T04:08:56,669 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-12T04:08:56,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=22 2024-11-12T04:08:56,673 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:08:56,676 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=22, resume processing ppid=21 2024-11-12T04:08:56,677 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-12T04:08:56,677 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=22, ppid=21, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 299 msec 2024-11-12T04:08:56,680 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=21, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 456 msec 2024-11-12T04:08:56,700 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x339cf9bd to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3a22c2aa 2024-11-12T04:08:56,711 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:08:56,711 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:08:56,712 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@621dbb47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:08:56,713 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:08:56,722 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:08:56,723 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:08:56,723 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f80018 connected 2024-11-12T04:08:56,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=21 2024-11-12T04:08:56,851 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:08:56,852 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:08:56,858 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-12T04:08:56,858 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-12T04:08:56,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=24, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:08:56,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=23, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:08:56,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:08:56,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-12T04:08:56,866 DEBUG [PEWorker-1 {}] procedure.MasterProcedureScheduler(583): PEER '1', shared lock count=0, exclusively locked by procId=24 2024-11-12T04:08:56,866 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1470): LOCK_EVENT_WAIT pid=23, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:08:56,868 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-12T04:08:56,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-12T04:08:56,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-12T04:08:56,877 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-12T04:08:56,879 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=25, ppid=24, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:08:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:08:56,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-12T04:08:57,031 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:08:57,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=25 2024-11-12T04:08:57,033 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-12T04:08:57,033 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-12T04:08:57,033 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-12T04:08:57,033 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:08:57,034 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f80018, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:08:57,034 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f80018, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:08:57,144 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f80018, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:08:57,144 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=25}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x339cf9bd to 127.0.0.1:50727 2024-11-12T04:08:57,145 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=25}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:08:57,146 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-12T04:08:57,161 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=25}] zookeeper.ZKUtil(111): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1/354fa2a6db59%2C40009%2C1731384526494.1731384529497 2024-11-12T04:08:57,172 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1/354fa2a6db59%2C40009%2C1731384526494.1731384529497 2024-11-12T04:08:57,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1 2024-11-12T04:08:57,173 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1 2024-11-12T04:08:57,173 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=25}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-12T04:08:57,173 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=25}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=25 2024-11-12T04:08:57,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=25 2024-11-12T04:08:57,176 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:08:57,180 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=25, resume processing ppid=24 2024-11-12T04:08:57,180 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=25, ppid=24, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 297 msec 2024-11-12T04:08:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:08:57,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-12T04:08:57,195 DEBUG [PEWorker-3 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-12T04:08:57,195 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-12T04:08:57,198 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=24, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 336 msec 2024-11-12T04:08:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:08:57,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=24 2024-11-12T04:08:57,499 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-12T04:08:57,499 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-12T04:08:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=26, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:08:57,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-11-12T04:08:57,504 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:08:57,513 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=26, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:08:57,515 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=26, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=14 msec 2024-11-12T04:08:57,515 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:08:57,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=26 2024-11-12T04:08:57,610 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-12T04:08:57,612 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(138): Process identifier=testing utility connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:08:57,623 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): testing utility0x0, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:08:57,623 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): testing utility-0x1012e84e2f80019 connected 2024-11-12T04:08:57,648 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[0] Thread=223 (was 222) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-2 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ForkJoinPool.commonPool-worker-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkUntil(LockSupport.java:410) java.base@17.0.11/java.util.concurrent.ForkJoinPool.awaitWork(ForkJoinPool.java:1726) java.base@17.0.11/java.util.concurrent.ForkJoinPool.runWorker(ForkJoinPool.java:1623) java.base@17.0.11/java.util.concurrent.ForkJoinWorkerThread.run(ForkJoinWorkerThread.java:165) Potentially hanging thread: RS-EventLoopGroup-1-3 app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=452 (was 448) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=336 (was 348), ProcessCount=11 (was 11), AvailableMemoryMB=2521 (was 2620) 2024-11-12T04:08:57,658 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[0] Thread=223, OpenFileDescriptor=452, MaxFileDescriptor=1048576, SystemLoadAverage=336, ProcessCount=11, AvailableMemoryMB=2521 2024-11-12T04:08:57,659 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-12T04:08:57,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=27, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-12T04:08:57,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-12T04:08:57,678 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:08:57,679 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:08:57,689 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:08:57,690 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8001a connected 2024-11-12T04:08:57,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-12T04:08:57,797 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8001a, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:08:57,821 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:08:57,976 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:08:57,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=28 2024-11-12T04:08:57,978 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-12T04:08:57,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-12T04:08:57,999 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=28}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:08:58,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:08:58,013 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:08:58,111 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-12T04:08:58,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=28 2024-11-12T04:08:58,114 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:08:58,117 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=28, resume processing ppid=27 2024-11-12T04:08:58,117 INFO [PEWorker-3 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-12T04:08:58,117 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=28, ppid=27, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 293 msec 2024-11-12T04:08:58,120 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=27, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 458 msec 2024-11-12T04:08:58,138 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6ce8419d to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4e1da0ea 2024-11-12T04:08:58,148 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:08:58,148 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:08:58,148 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@52139f93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:08:58,149 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:08:58,160 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:08:58,161 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f8001c connected 2024-11-12T04:08:58,161 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:08:58,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=27 2024-11-12T04:08:58,288 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-12T04:08:58,289 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-12T04:08:58,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=29, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:08:58,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=29 2024-11-12T04:08:58,305 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:08:58,306 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:08:58,314 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:08:58,314 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8001d connected 2024-11-12T04:08:58,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=29 2024-11-12T04:08:58,422 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8001d, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:08:58,432 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:08:58,586 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:08:58,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=30 2024-11-12T04:08:58,587 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:08:58,590 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-12T04:08:58,590 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-12T04:08:58,591 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:08:58,591 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f8001c, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:08:58,591 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f8001c, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:08:58,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=29 2024-11-12T04:08:58,702 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f8001c, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:08:58,702 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=30}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6ce8419d to 127.0.0.1:50727 2024-11-12T04:08:58,702 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=30}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:08:58,720 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:08:58,720 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-12T04:08:58,721 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=30}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=30 2024-11-12T04:08:58,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=30 2024-11-12T04:08:58,724 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:08:58,727 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=30, resume processing ppid=29 2024-11-12T04:08:58,727 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=30, ppid=29, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 292 msec 2024-11-12T04:08:58,727 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-12T04:08:58,729 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=29, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 438 msec 2024-11-12T04:08:58,748 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3b19d624 to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2985a32c 2024-11-12T04:08:58,756 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:08:58,757 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:08:58,757 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@7706fb2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:08:58,758 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:08:58,769 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:08:58,769 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:08:58,769 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f8001f connected 2024-11-12T04:08:58,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=29 2024-11-12T04:08:58,921 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:08:58,922 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:08:58,924 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-11-12T04:08:58,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=31, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:08:58,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=31 2024-11-12T04:08:58,943 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:08:58,944 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:08:58,952 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:08:58,953 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80020 connected 2024-11-12T04:08:59,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:08:59,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=31 2024-11-12T04:08:59,061 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80020, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:08:59,075 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:08:59,230 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:08:59,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=32 2024-11-12T04:08:59,232 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:08:59,235 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-12T04:08:59,235 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-12T04:08:59,235 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:08:59,235 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f8001f, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:08:59,236 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f8001f, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:08:59,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=31 2024-11-12T04:08:59,344 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f8001f, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:08:59,344 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=32}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3b19d624 to 127.0.0.1:50727 2024-11-12T04:08:59,344 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=32}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:08:59,364 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:08:59,364 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-12T04:08:59,365 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=32}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=32 2024-11-12T04:08:59,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=32 2024-11-12T04:08:59,368 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:08:59,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=32, resume processing ppid=31 2024-11-12T04:08:59,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=32, ppid=31, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 294 msec 2024-11-12T04:08:59,370 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-11-12T04:08:59,372 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=31, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 446 msec 2024-11-12T04:08:59,391 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x51bcf5b0 to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@168dc387 2024-11-12T04:08:59,402 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:08:59,403 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:08:59,403 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@5b4243bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:08:59,404 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:08:59,414 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:08:59,415 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:08:59,415 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f80022 connected 2024-11-12T04:08:59,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=31 2024-11-12T04:08:59,558 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:08:59,559 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:08:59,560 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns1:testNamespacesAndTableCfsConfigConflict1=null},bandwidth=0,serial=false 2024-11-12T04:08:59,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=33, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:08:59,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-12T04:08:59,581 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:08:59,582 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:08:59,589 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:08:59,589 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80023 connected 2024-11-12T04:08:59,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-12T04:08:59,698 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80023, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:08:59,698 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:08:59,700 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=33, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:08:59,702 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=33, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=139 msec 2024-11-12T04:08:59,702 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:08:59,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=33 2024-11-12T04:08:59,879 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns1:testNamespacesAndTableCfsConfigConflict1 is conflict with namespaces ns1 in peer config 2024-11-12T04:08:59,880 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:08:59,882 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-11-12T04:08:59,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=34, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:08:59,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-11-12T04:08:59,904 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:08:59,905 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:08:59,947 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:08:59,948 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80024 connected 2024-11-12T04:08:59,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-11-12T04:09:00,056 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80024, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:00,070 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:00,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-11-12T04:09:00,223 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:00,224 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=35 2024-11-12T04:09:00,224 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:09:00,225 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-12T04:09:00,226 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-12T04:09:00,226 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:00,226 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f80022, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:00,226 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f80022, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:00,336 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f80022, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:00,336 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=35}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x51bcf5b0 to 127.0.0.1:50727 2024-11-12T04:09:00,336 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=35}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:00,360 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:00,361 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-12T04:09:00,362 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-12T04:09:00,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=35 2024-11-12T04:09:00,368 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:00,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=35, resume processing ppid=34 2024-11-12T04:09:00,371 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=35, ppid=34, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 298 msec 2024-11-12T04:09:00,371 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-11-12T04:09:00,373 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=34, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 489 msec 2024-11-12T04:09:00,391 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x29d49318 to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@126a7de7 2024-11-12T04:09:00,402 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:00,403 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:00,403 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@44945f1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:00,404 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:00,414 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:00,414 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:00,415 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f80026 connected 2024-11-12T04:09:00,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=34 2024-11-12T04:09:00,509 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:09:00,510 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:00,512 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2],tableCFs={ns2:testNamespacesAndTableCfsConfigConflict2=null},bandwidth=0,serial=false 2024-11-12T04:09:00,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=36, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:09:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-12T04:09:00,533 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:00,533 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:00,544 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:00,544 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80027 connected 2024-11-12T04:09:00,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-12T04:09:00,652 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80027, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:00,653 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:00,658 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=36, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkNamespacesAndTableCfsConfigConflict(ReplicationPeerManager.java:369) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:331) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preUpdatePeerConfig(ReplicationPeerManager.java:161) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure.prePeerModification(UpdatePeerConfigProcedure.java:176) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:00,663 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=36, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-update_config-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure exec-time=147 msec 2024-11-12T04:09:00,663 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:00,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=36 2024-11-12T04:09:00,840 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 failed with Table-cfs ns2:testNamespacesAndTableCfsConfigConflict2 is conflict with namespaces ns2 in peer config 2024-11-12T04:09:00,841 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-12T04:09:00,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=37, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:00,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=37 2024-11-12T04:09:00,850 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-12T04:09:00,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-12T04:09:00,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-12T04:09:00,860 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-12T04:09:00,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:00,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=37 2024-11-12T04:09:01,015 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:01,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=38 2024-11-12T04:09:01,016 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-12T04:09:01,017 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-12T04:09:01,017 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-12T04:09:01,017 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:01,018 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f80026, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:01,019 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f80026, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:01,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:09:01,131 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f80026, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:01,131 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=38}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x29d49318 to 127.0.0.1:50727 2024-11-12T04:09:01,131 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=38}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:01,131 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-12T04:09:01,140 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=38}] zookeeper.ZKUtil(111): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1/354fa2a6db59%2C40009%2C1731384526494.1731384529497 2024-11-12T04:09:01,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1/354fa2a6db59%2C40009%2C1731384526494.1731384529497 2024-11-12T04:09:01,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1 2024-11-12T04:09:01,152 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1 2024-11-12T04:09:01,153 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=38}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-12T04:09:01,153 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=38}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=38 2024-11-12T04:09:01,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=38 2024-11-12T04:09:01,157 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:01,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=37 2024-11-12T04:09:01,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=38, resume processing ppid=37 2024-11-12T04:09:01,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=38, ppid=37, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 295 msec 2024-11-12T04:09:01,169 DEBUG [PEWorker-2 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-12T04:09:01,169 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-12T04:09:01,172 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=37, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 328 msec 2024-11-12T04:09:01,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=37 2024-11-12T04:09:01,468 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-12T04:09:01,469 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-12T04:09:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=39, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:01,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-12T04:09:01,472 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:01,474 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=39, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:01,476 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=39, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec 2024-11-12T04:09:01,476 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:01,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=39 2024-11-12T04:09:01,580 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-12T04:09:01,581 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-12T04:09:01,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=40, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:01,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=40 2024-11-12T04:09:01,585 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:01,587 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=40, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:01,589 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=40, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec 2024-11-12T04:09:01,589 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:01,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=40 2024-11-12T04:09:01,689 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-12T04:09:01,705 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testNamespacesAndTableCfsConfigConflict[0] Thread=223 (was 223), OpenFileDescriptor=450 (was 452), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=309 (was 336), ProcessCount=11 (was 11), AvailableMemoryMB=2500 (was 2521) 2024-11-12T04:09:01,724 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAddRemovePeer[0] Thread=223, OpenFileDescriptor=450, MaxFileDescriptor=1048576, SystemLoadAverage=309, ProcessCount=11, AvailableMemoryMB=2499 2024-11-12T04:09:01,725 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-12T04:09:01,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=41, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-12T04:09:01,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41 2024-11-12T04:09:01,750 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:01,751 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:01,798 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:01,798 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80028 connected 2024-11-12T04:09:01,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41 2024-11-12T04:09:01,906 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80028, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:01,930 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=42, ppid=41, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:02,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41 2024-11-12T04:09:02,083 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:02,084 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=42 2024-11-12T04:09:02,084 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=42}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-12T04:09:02,106 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=42}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:02,127 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=42}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:02,172 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=42}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=42 2024-11-12T04:09:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=42 2024-11-12T04:09:02,176 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:02,180 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=42, resume processing ppid=41 2024-11-12T04:09:02,180 INFO [PEWorker-4 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-12T04:09:02,180 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=42, ppid=41, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 246 msec 2024-11-12T04:09:02,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=41, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 455 msec 2024-11-12T04:09:02,211 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x760b7ebb to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@371cd07a 2024-11-12T04:09:02,223 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:02,224 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:02,224 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@7907e9de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:02,225 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:02,235 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:02,236 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:02,236 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f8002a connected 2024-11-12T04:09:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=41 2024-11-12T04:09:02,358 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-12T04:09:02,359 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-12T04:09:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=43, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-12T04:09:02,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=43 2024-11-12T04:09:02,374 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:02,374 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:02,406 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:02,406 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8002b connected 2024-11-12T04:09:02,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=43 2024-11-12T04:09:02,519 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8002b, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:02,519 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:121) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:02,521 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=43, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:121) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:02,522 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=43, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Replication peer 1 already exists; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=162 msec 2024-11-12T04:09:02,522 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:02,648 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T04:09:02,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=43 2024-11-12T04:09:02,679 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Replication peer 1 already exists 2024-11-12T04:09:02,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-11-12T04:09:02,685 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-12T04:09:02,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=44, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=44 2024-11-12T04:09:02,687 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:02,688 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=44, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:02,689 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=44, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-11-12T04:09:02,690 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:02,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=44 2024-11-12T04:09:02,800 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-12T04:09:02,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-11-12T04:09:02,804 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=2, config=clusterKey=127.0.0.1:50727:/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-12T04:09:02,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=45, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-12T04:09:02,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-12T04:09:02,824 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:02,824 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:02,864 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:02,865 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8002c connected 2024-11-12T04:09:02,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-12T04:09:02,973 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8002c, quorum=127.0.0.1:50727, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:02,996 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:03,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-12T04:09:03,150 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:03,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=46 2024-11-12T04:09:03,152 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=2, type=ADD_PEER 2024-11-12T04:09:03,169 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=46}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:03,181 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.ReplicationSource(229): queueId=2, ReplicationSource: 2, currentBandwidth=0 2024-11-12T04:09:03,206 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=46}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=46 2024-11-12T04:09:03,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=46 2024-11-12T04:09:03,210 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 2 for ADD on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:03,213 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=46, resume processing ppid=45 2024-11-12T04:09:03,213 INFO [PEWorker-3 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 2, config clusterKey=127.0.0.1:50727:/hbase-test2,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-12T04:09:03,213 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=46, ppid=45, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 215 msec 2024-11-12T04:09:03,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=45, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 409 msec 2024-11-12T04:09:03,236 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 2000 millis 2024-11-12T04:09:03,237 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,2 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7299612d to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@552c479d 2024-11-12T04:09:03,248 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,2 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test2/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:03,248 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,2 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:03,249 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@2384ffa5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:03,250 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 2 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:03,260 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,2-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 20x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:03,261 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,2 {}] regionserver.ReplicationSource(569): peerId=2, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:03,261 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 2-0x1012e84e2f8002e connected 2024-11-12T04:09:03,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=45 2024-11-12T04:09:03,439 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 2 completed 2024-11-12T04:09:03,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-11-12T04:09:03,442 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-12T04:09:03,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=47, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:03,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-11-12T04:09:03,449 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-12T04:09:03,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-12T04:09:03,460 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-12T04:09:03,461 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-12T04:09:03,462 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=48, ppid=47, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:03,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-11-12T04:09:03,615 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:03,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=48 2024-11-12T04:09:03,617 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-12T04:09:03,617 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-12T04:09:03,617 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-12T04:09:03,617 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:03,617 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f8002a, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:03,618 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f8002a, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:03,727 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f8002a, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:03,728 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=48}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x760b7ebb to 127.0.0.1:50727 2024-11-12T04:09:03,728 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=48}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:03,728 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-12T04:09:03,741 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=48}] zookeeper.ZKUtil(111): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1/354fa2a6db59%2C40009%2C1731384526494.1731384529497 2024-11-12T04:09:03,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1/354fa2a6db59%2C40009%2C1731384526494.1731384529497 2024-11-12T04:09:03,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1 2024-11-12T04:09:03,752 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1 2024-11-12T04:09:03,752 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=48}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-12T04:09:03,752 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=48}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=48 2024-11-12T04:09:03,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=48 2024-11-12T04:09:03,755 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:03,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=48, resume processing ppid=47 2024-11-12T04:09:03,757 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=48, ppid=47, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 293 msec 2024-11-12T04:09:03,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-11-12T04:09:03,777 DEBUG [PEWorker-4 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-12T04:09:03,777 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-12T04:09:03,779 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=47, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 335 msec 2024-11-12T04:09:04,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=47 2024-11-12T04:09:04,069 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-12T04:09:04,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-11-12T04:09:04,070 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-12T04:09:04,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=49, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:04,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-11-12T04:09:04,074 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/2/peer-state 2024-11-12T04:09:04,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2/peer-state 2024-11-12T04:09:04,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/2 2024-11-12T04:09:04,085 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/2 2024-11-12T04:09:04,086 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:04,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-11-12T04:09:04,239 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:04,241 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=50 2024-11-12T04:09:04,241 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=2, type=REMOVE_PEER 2024-11-12T04:09:04,241 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 2: 0 2024-11-12T04:09:04,241 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.ReplicationSource(657): peerId=2, Closing source 2 because: Replication stream was removed by a user 2024-11-12T04:09:04,242 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,2 {}] regionserver.ReplicationSource(508): peerId=2, Interrupted while sleeping between retries 2024-11-12T04:09:04,242 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,2 {}] zookeeper.ZKUtil(165): connection to cluster: 2-0x1012e84e2f8002e, quorum=127.0.0.1:50727, baseZNode=/hbase-test2 Unable to set watcher on znode (/hbase-test2/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:04,243 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,2 {}] zookeeper.ZKWatcher(734): connection to cluster: 2-0x1012e84e2f8002e, quorum=127.0.0.1:50727, baseZNode=/hbase-test2 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:04,352 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,2-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 2-0x1012e84e2f8002e, quorum=127.0.0.1:50727, baseZNode=/hbase-test2 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:04,352 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=50}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7299612d to 127.0.0.1:50727 2024-11-12T04:09:04,352 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=50}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:04,353 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.ReplicationSourceManager(430): Done with the queue 2 2024-11-12T04:09:04,361 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=50}] zookeeper.ZKUtil(111): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/2/354fa2a6db59%2C40009%2C1731384526494.1731384529497 2024-11-12T04:09:04,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/2/354fa2a6db59%2C40009%2C1731384526494.1731384529497 2024-11-12T04:09:04,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/2 2024-11-12T04:09:04,372 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/2 2024-11-12T04:09:04,373 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=50}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/2 not found in hfile reference queue. 2024-11-12T04:09:04,373 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=50}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=50 2024-11-12T04:09:04,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=50 2024-11-12T04:09:04,376 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 2 for REMOVE on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:04,378 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=50, resume processing ppid=49 2024-11-12T04:09:04,378 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=50, ppid=49, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 290 msec 2024-11-12T04:09:04,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-11-12T04:09:04,390 DEBUG [PEWorker-5 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/2 not found in hfile reference queue. 2024-11-12T04:09:04,390 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(76): Successfully removed peer 2 2024-11-12T04:09:04,392 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=49, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 320 msec 2024-11-12T04:09:04,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=49 2024-11-12T04:09:04,699 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 2 completed 2024-11-12T04:09:04,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-11-12T04:09:04,701 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-12T04:09:04,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=51, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:04,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-12T04:09:04,706 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:04,708 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=51, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:04,709 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=51, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=7 msec 2024-11-12T04:09:04,709 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:04,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=51 2024-11-12T04:09:04,809 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-12T04:09:04,809 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-12T04:09:04,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=52, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:04,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=52 2024-11-12T04:09:04,812 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:04,813 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=52, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:04,814 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=52, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-11-12T04:09:04,815 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:04,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=52 2024-11-12T04:09:04,919 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-12T04:09:04,937 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAddRemovePeer[0] Thread=221 (was 223), OpenFileDescriptor=444 (was 450), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=284 (was 309), ProcessCount=11 (was 11), AvailableMemoryMB=2446 (was 2499) 2024-11-12T04:09:04,947 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[0] Thread=221, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=284, ProcessCount=11, AvailableMemoryMB=2445 2024-11-12T04:09:04,949 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=whatever,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-12T04:09:04,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=53, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-12T04:09:04,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53 2024-11-12T04:09:04,952 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:291) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:288) ~[classes/:?] ... 12 more 2024-11-12T04:09:04,954 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=53, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:291) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more Caused by: java.lang.ClassNotFoundException: whatever at jdk.internal.loader.BuiltinClassLoader.loadClass(BuiltinClassLoader.java:641) ~[?:?] at jdk.internal.loader.ClassLoaders$AppClassLoader.loadClass(ClassLoaders.java:188) ~[?:?] at java.lang.ClassLoader.loadClass(ClassLoader.java:525) ~[?:?] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:288) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:04,956 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=53, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Can not instantiate configured replication endpoint class=whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=5 msec 2024-11-12T04:09:04,956 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:05,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:09:05,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=53 2024-11-12T04:09:05,060 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Can not instantiate configured replication endpoint class=whatever 2024-11-12T04:09:05,069 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-12T04:09:05,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=54, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:05,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-12T04:09:05,073 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:05,074 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=54, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:05,075 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=54, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-11-12T04:09:05,075 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=54 2024-11-12T04:09:05,179 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-12T04:09:05,180 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-12T04:09:05,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=55, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:05,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-11-12T04:09:05,184 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:05,185 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=55, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:05,187 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=55, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec 2024-11-12T04:09:05,187 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:05,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=55 2024-11-12T04:09:05,289 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-12T04:09:05,301 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidReplicationEndpoint[0] Thread=221 (was 221), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=284 (was 284), ProcessCount=11 (was 11), AvailableMemoryMB=2446 (was 2445) - AvailableMemoryMB LEAK? - 2024-11-12T04:09:05,309 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[0] Thread=221, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=284, ProcessCount=11, AvailableMemoryMB=2445 2024-11-12T04:09:05,310 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=whatever,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-12T04:09:05,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=56, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-12T04:09:05,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=56 2024-11-12T04:09:05,314 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] ... 13 more 2024-11-12T04:09:05,315 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=56, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more Caused by: java.io.IOException: Cluster key passed whatever is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:05,317 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=56, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: whatever; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=5 msec 2024-11-12T04:09:05,317 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:05,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=56 2024-11-12T04:09:05,420 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Invalid cluster key: whatever 2024-11-12T04:09:05,421 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-12T04:09:05,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=57, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:05,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-12T04:09:05,424 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:05,425 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=57, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:05,427 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=57, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-11-12T04:09:05,427 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:05,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=57 2024-11-12T04:09:05,529 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-12T04:09:05,530 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-12T04:09:05,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=58, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:05,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=58 2024-11-12T04:09:05,534 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:05,537 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=58, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:05,539 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=58, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=7 msec 2024-11-12T04:09:05,539 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=58 2024-11-12T04:09:05,638 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-12T04:09:05,650 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testInvalidClusterKey[0] Thread=221 (was 221), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=284 (was 284), ProcessCount=11 (was 11), AvailableMemoryMB=2442 (was 2445) 2024-11-12T04:09:05,658 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[0] Thread=221, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=284, ProcessCount=11, AvailableMemoryMB=2441 2024-11-12T04:09:05,659 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-12T04:09:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=59, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-12T04:09:05,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-11-12T04:09:05,674 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:05,675 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:05,722 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:05,723 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8002f connected 2024-11-12T04:09:05,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-11-12T04:09:05,831 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8002f, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:05,855 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:05,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-11-12T04:09:06,009 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:06,010 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=60 2024-11-12T04:09:06,010 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-12T04:09:06,024 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=60}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:06,036 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:06,077 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=60}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=60 2024-11-12T04:09:06,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=60 2024-11-12T04:09:06,082 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:06,087 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=60, resume processing ppid=59 2024-11-12T04:09:06,087 INFO [PEWorker-1 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-12T04:09:06,087 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=60, ppid=59, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 229 msec 2024-11-12T04:09:06,089 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=59, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 428 msec 2024-11-12T04:09:06,114 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x18ec5d48 to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2fc4996d 2024-11-12T04:09:06,149 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:06,150 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:06,150 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3d8a7a9b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:06,151 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:06,164 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:06,165 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:06,165 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f80031 connected 2024-11-12T04:09:06,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=59 2024-11-12T04:09:06,290 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-12T04:09:06,291 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-12T04:09:06,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=61, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:09:06,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-11-12T04:09:06,311 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:06,312 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:06,322 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:06,323 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80032 connected 2024-11-12T04:09:06,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-11-12T04:09:06,431 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80032, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:06,440 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=62, ppid=61, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:06,592 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:06,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=62 2024-11-12T04:09:06,593 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:09:06,595 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-12T04:09:06,595 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-12T04:09:06,595 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:06,595 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f80031, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:06,596 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f80031, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-11-12T04:09:06,706 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f80031, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:06,706 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=62}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x18ec5d48 to 127.0.0.1:50727 2024-11-12T04:09:06,707 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=62}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:06,727 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:06,727 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-12T04:09:06,728 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=62}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=62 2024-11-12T04:09:06,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=62 2024-11-12T04:09:06,730 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:06,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=62, resume processing ppid=61 2024-11-12T04:09:06,733 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-12T04:09:06,733 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=62, ppid=61, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 290 msec 2024-11-12T04:09:06,734 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=61, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 442 msec 2024-11-12T04:09:06,755 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x34ccbc4c to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@37162941 2024-11-12T04:09:06,765 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:06,765 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:06,765 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@38ffac28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:06,766 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:06,799 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:06,800 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:06,800 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f80034 connected 2024-11-12T04:09:06,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=61 2024-11-12T04:09:06,929 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:09:06,929 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:06,931 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-11-12T04:09:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=63, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:09:06,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-12T04:09:06,953 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:06,953 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:06,964 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:06,965 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80035 connected 2024-11-12T04:09:07,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-12T04:09:07,073 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80035, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:07,086 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:07,238 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:07,239 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=64 2024-11-12T04:09:07,240 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:09:07,242 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-12T04:09:07,242 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-12T04:09:07,242 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:07,242 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f80034, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:07,243 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f80034, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-12T04:09:07,352 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f80034, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:07,352 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=64}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x34ccbc4c to 127.0.0.1:50727 2024-11-12T04:09:07,353 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=64}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:07,370 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:07,371 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-12T04:09:07,371 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=64}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=64 2024-11-12T04:09:07,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=64 2024-11-12T04:09:07,374 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:07,376 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=64, resume processing ppid=63 2024-11-12T04:09:07,376 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns2, ns1],bandwidth=0,serial=false 2024-11-12T04:09:07,376 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=64, ppid=63, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 288 msec 2024-11-12T04:09:07,378 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=63, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 445 msec 2024-11-12T04:09:07,399 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x6b4b2829 to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@483fac5a 2024-11-12T04:09:07,411 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:07,411 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:07,412 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@7f81d218, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:07,413 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:07,422 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:07,423 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:07,423 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f80037 connected 2024-11-12T04:09:07,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=63 2024-11-12T04:09:07,558 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:09:07,559 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:07,559 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:07,560 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-11-12T04:09:07,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=65, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:09:07,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-11-12T04:09:07,575 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:07,575 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:07,585 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:07,586 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80038 connected 2024-11-12T04:09:07,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-11-12T04:09:07,694 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80038, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:07,707 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=66, ppid=65, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:07,859 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:07,860 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=66 2024-11-12T04:09:07,860 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:09:07,861 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-12T04:09:07,861 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-12T04:09:07,861 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:07,862 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f80037, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:07,862 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f80037, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:07,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-11-12T04:09:07,973 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f80037, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:07,973 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=66}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x6b4b2829 to 127.0.0.1:50727 2024-11-12T04:09:07,973 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=66}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:07,993 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:07,994 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-12T04:09:07,994 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=66}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=66 2024-11-12T04:09:07,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=66 2024-11-12T04:09:07,996 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:07,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=66, resume processing ppid=65 2024-11-12T04:09:07,998 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,namespaces=[ns1],bandwidth=0,serial=false 2024-11-12T04:09:07,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=66, ppid=65, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 289 msec 2024-11-12T04:09:08,000 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=65, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 438 msec 2024-11-12T04:09:08,021 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x3c6c7fe5 to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@311dd8a 2024-11-12T04:09:08,031 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:08,032 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:08,032 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@6d545b8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:08,033 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:08,043 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:08,044 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:08,044 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f8003a connected 2024-11-12T04:09:08,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=65 2024-11-12T04:09:08,187 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:09:08,188 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:08,189 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-12T04:09:08,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=67, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:08,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67 2024-11-12T04:09:08,193 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-12T04:09:08,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-12T04:09:08,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-12T04:09:08,202 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-12T04:09:08,203 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:08,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67 2024-11-12T04:09:08,355 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:08,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=68 2024-11-12T04:09:08,356 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-12T04:09:08,356 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-12T04:09:08,356 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-12T04:09:08,356 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:08,356 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f8003a, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:08,357 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f8003a, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:08,465 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f8003a, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:08,465 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=68}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x3c6c7fe5 to 127.0.0.1:50727 2024-11-12T04:09:08,465 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=68}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:08,465 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-12T04:09:08,474 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=68}] zookeeper.ZKUtil(111): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1/354fa2a6db59%2C40009%2C1731384526494.1731384529497 2024-11-12T04:09:08,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1/354fa2a6db59%2C40009%2C1731384526494.1731384529497 2024-11-12T04:09:08,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1 2024-11-12T04:09:08,486 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1 2024-11-12T04:09:08,486 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=68}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-12T04:09:08,486 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=68}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=68 2024-11-12T04:09:08,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=68 2024-11-12T04:09:08,490 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:08,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=68, resume processing ppid=67 2024-11-12T04:09:08,493 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=68, ppid=67, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 287 msec 2024-11-12T04:09:08,502 DEBUG [PEWorker-5 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-12T04:09:08,502 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-12T04:09:08,504 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=67, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 313 msec 2024-11-12T04:09:08,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=67 2024-11-12T04:09:08,508 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-12T04:09:08,509 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-12T04:09:08,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=69, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:08,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-12T04:09:08,510 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:08,512 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=69, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:08,512 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=69, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-11-12T04:09:08,513 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:08,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=69 2024-11-12T04:09:08,619 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-12T04:09:08,619 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-12T04:09:08,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=70, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:08,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=70 2024-11-12T04:09:08,622 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:08,624 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=70, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:08,625 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=70, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-11-12T04:09:08,625 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:08,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=70 2024-11-12T04:09:08,728 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-12T04:09:08,744 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetPeerNamespaces[0] Thread=220 (was 221), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=284 (was 284), ProcessCount=11 (was 11), AvailableMemoryMB=2425 (was 2441) 2024-11-12T04:09:08,750 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[0] Thread=220, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=284, ProcessCount=11, AvailableMemoryMB=2425 2024-11-12T04:09:08,752 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-12T04:09:08,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=71, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-12T04:09:08,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-11-12T04:09:08,769 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:08,770 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:08,781 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:08,781 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8003b connected 2024-11-12T04:09:08,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-11-12T04:09:08,889 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8003b, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:08,911 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=72, ppid=71, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:09,064 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:09,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=72 2024-11-12T04:09:09,066 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-12T04:09:09,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-11-12T04:09:09,080 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=72}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:09,093 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:09,135 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=72}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=72 2024-11-12T04:09:09,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=72 2024-11-12T04:09:09,138 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:09,141 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=72, resume processing ppid=71 2024-11-12T04:09:09,141 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=72, ppid=71, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 227 msec 2024-11-12T04:09:09,141 INFO [PEWorker-3 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-12T04:09:09,143 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=71, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 389 msec 2024-11-12T04:09:09,165 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x28504ca9 to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@10fe6fcf 2024-11-12T04:09:09,177 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:09,178 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:09,178 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3b0f50f5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:09,179 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:09,189 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:09,190 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:09,190 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f8003d connected 2024-11-12T04:09:09,232 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T04:09:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=71 2024-11-12T04:09:09,378 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-12T04:09:09,378 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-12T04:09:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=73, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:09:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-11-12T04:09:09,398 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:09,398 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:09,406 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:09,406 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8003e connected 2024-11-12T04:09:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-11-12T04:09:09,514 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8003e, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:09,527 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:09,679 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:09,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=74 2024-11-12T04:09:09,680 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:09:09,681 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-12T04:09:09,681 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-12T04:09:09,682 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:09,682 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f8003d, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:09,682 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f8003d, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-11-12T04:09:09,789 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f8003d, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:09,789 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=74}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x28504ca9 to 127.0.0.1:50727 2024-11-12T04:09:09,789 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=74}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:09,809 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:09,810 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-12T04:09:09,810 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=74}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=74 2024-11-12T04:09:09,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=74 2024-11-12T04:09:09,813 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:09,816 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=74, resume processing ppid=73 2024-11-12T04:09:09,816 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-12T04:09:09,816 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=74, ppid=73, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 286 msec 2024-11-12T04:09:09,818 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=73, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 438 msec 2024-11-12T04:09:09,852 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x5300cba4 to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@765e9241 2024-11-12T04:09:09,860 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:09,861 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:09,861 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@7db8ba6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:09,863 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:09,873 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:09,873 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:09,873 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f80040 connected 2024-11-12T04:09:10,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=73 2024-11-12T04:09:10,008 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:09:10,009 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:10,010 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-11-12T04:09:10,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=75, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:09:10,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-12T04:09:10,029 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:10,029 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:10,039 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:10,039 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80041 connected 2024-11-12T04:09:10,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-12T04:09:10,148 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80041, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:10,162 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=76, ppid=75, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:10,315 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:10,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=76 2024-11-12T04:09:10,316 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:09:10,317 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-12T04:09:10,317 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-12T04:09:10,317 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:10,318 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f80040, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:10,318 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f80040, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:10,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-12T04:09:10,427 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f80040, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:10,427 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=76}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x5300cba4 to 127.0.0.1:50727 2024-11-12T04:09:10,427 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=76}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:10,445 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:10,445 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-12T04:09:10,446 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=76}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=76 2024-11-12T04:09:10,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=76 2024-11-12T04:09:10,449 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:10,453 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=76, resume processing ppid=75 2024-11-12T04:09:10,453 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=76, ppid=75, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 287 msec 2024-11-12T04:09:10,453 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null},bandwidth=0,serial=false 2024-11-12T04:09:10,458 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=75, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 442 msec 2024-11-12T04:09:10,486 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x4e45551d to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2a424074 2024-11-12T04:09:10,498 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:10,499 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:10,499 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@2d8d147f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:10,501 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:10,510 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:10,510 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:10,511 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f80043 connected 2024-11-12T04:09:10,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=75 2024-11-12T04:09:10,640 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:09:10,641 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:10,643 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:10,644 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-11-12T04:09:10,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=77, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:09:10,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-11-12T04:09:10,664 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:10,664 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:10,708 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:10,709 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80044 connected 2024-11-12T04:09:10,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-11-12T04:09:10,827 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80044, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:10,841 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:10,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-11-12T04:09:10,995 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:10,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=78 2024-11-12T04:09:10,996 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:09:10,998 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-12T04:09:10,998 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-12T04:09:10,998 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:10,998 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f80043, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:10,999 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f80043, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:11,110 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f80043, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:11,111 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=78}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x4e45551d to 127.0.0.1:50727 2024-11-12T04:09:11,111 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=78}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:11,128 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:11,129 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-12T04:09:11,129 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=78}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=78 2024-11-12T04:09:11,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=78 2024-11-12T04:09:11,131 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:11,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=78, resume processing ppid=77 2024-11-12T04:09:11,133 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=78, ppid=77, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 290 msec 2024-11-12T04:09:11,133 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-11-12T04:09:11,135 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=77, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 489 msec 2024-11-12T04:09:11,156 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x2e57be12 to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@423ac075 2024-11-12T04:09:11,169 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:11,170 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:11,170 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@56ab1a5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:11,171 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:11,181 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:11,181 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:11,181 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f80046 connected 2024-11-12T04:09:11,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=77 2024-11-12T04:09:11,278 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:09:11,279 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:11,281 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:11,282 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-11-12T04:09:11,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=79, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:09:11,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-11-12T04:09:11,303 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:11,303 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:11,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-11-12T04:09:11,538 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:11,539 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80047 connected 2024-11-12T04:09:11,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-11-12T04:09:11,652 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80047, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:11,666 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=80, ppid=79, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:11,820 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:11,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=80 2024-11-12T04:09:11,821 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:09:11,822 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-12T04:09:11,823 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-12T04:09:11,823 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:11,823 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f80046, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:11,823 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f80046, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:11,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-11-12T04:09:11,931 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f80046, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:11,931 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=80}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x2e57be12 to 127.0.0.1:50727 2024-11-12T04:09:11,931 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=80}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:11,944 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:11,944 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-12T04:09:11,944 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=80}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=80 2024-11-12T04:09:11,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=80 2024-11-12T04:09:11,947 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:11,949 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=80, resume processing ppid=79 2024-11-12T04:09:11,949 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=80, ppid=79, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 281 msec 2024-11-12T04:09:11,949 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null},bandwidth=0,serial=false 2024-11-12T04:09:11,950 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=79, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 667 msec 2024-11-12T04:09:11,971 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x25c2b18a to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2434ad45 2024-11-12T04:09:11,981 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:11,981 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:11,982 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@3a6043e5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:11,982 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:11,993 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:11,994 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:11,994 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f80049 connected 2024-11-12T04:09:12,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=79 2024-11-12T04:09:12,419 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:09:12,419 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:12,421 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:12,422 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-11-12T04:09:12,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=81, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:09:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-12T04:09:12,437 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:12,438 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:12,481 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:12,482 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8004a connected 2024-11-12T04:09:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-12T04:09:12,589 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8004a, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:12,603 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:12,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-12T04:09:12,756 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:12,757 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=82 2024-11-12T04:09:12,758 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:09:12,761 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-12T04:09:12,761 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-12T04:09:12,762 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:12,762 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f80049, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:12,763 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f80049, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:12,873 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f80049, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:12,873 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=82}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x25c2b18a to 127.0.0.1:50727 2024-11-12T04:09:12,873 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=82}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:12,894 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:12,894 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-12T04:09:12,894 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=82}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=82 2024-11-12T04:09:12,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=82 2024-11-12T04:09:12,897 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:12,899 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=82, resume processing ppid=81 2024-11-12T04:09:12,899 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=82, ppid=81, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 294 msec 2024-11-12T04:09:12,899 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-11-12T04:09:12,900 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=81, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 476 msec 2024-11-12T04:09:12,923 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x0bd10e58 to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@40f57838 2024-11-12T04:09:12,935 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:12,936 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:12,936 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@639b8371, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:12,937 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:12,947 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:12,948 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:12,948 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f8004c connected 2024-11-12T04:09:13,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=81 2024-11-12T04:09:13,048 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:09:13,049 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:13,050 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:13,051 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-11-12T04:09:13,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=83, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:09:13,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-11-12T04:09:13,067 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:13,068 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:13,076 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:13,077 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8004d connected 2024-11-12T04:09:13,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-11-12T04:09:13,185 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8004d, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:13,195 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:13,348 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:13,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=84 2024-11-12T04:09:13,349 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:09:13,350 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-12T04:09:13,350 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-12T04:09:13,350 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:13,351 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f8004c, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:13,351 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f8004c, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:13,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-11-12T04:09:13,460 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f8004c, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:13,460 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=84}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x0bd10e58 to 127.0.0.1:50727 2024-11-12T04:09:13,461 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=84}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:13,482 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:13,482 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-12T04:09:13,482 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=84}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=84 2024-11-12T04:09:13,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=84 2024-11-12T04:09:13,485 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:13,487 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=84, resume processing ppid=83 2024-11-12T04:09:13,487 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-11-12T04:09:13,487 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=84, ppid=83, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 290 msec 2024-11-12T04:09:13,488 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=83, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 436 msec 2024-11-12T04:09:13,509 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x362d02b4 to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@37469f85 2024-11-12T04:09:13,519 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:13,519 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:13,519 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@7c1cc66c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:13,520 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:13,531 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:13,531 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:13,531 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f8004f connected 2024-11-12T04:09:13,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=83 2024-11-12T04:09:13,678 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:09:13,679 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:13,681 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-11-12T04:09:13,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=85, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:09:13,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-11-12T04:09:13,702 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:13,703 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:13,714 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:13,715 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80050 connected 2024-11-12T04:09:13,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-11-12T04:09:13,823 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80050, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:13,832 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:13,984 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:13,985 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=86 2024-11-12T04:09:13,985 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:09:13,986 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=86}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=86 2024-11-12T04:09:13,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=86 2024-11-12T04:09:13,989 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:13,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=86, resume processing ppid=85 2024-11-12T04:09:13,991 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=86, ppid=85, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 157 msec 2024-11-12T04:09:13,991 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2]},bandwidth=0,serial=false 2024-11-12T04:09:13,993 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=85, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 310 msec 2024-11-12T04:09:13,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=85 2024-11-12T04:09:13,998 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:09:13,998 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:13,999 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:14,000 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-11-12T04:09:14,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=87, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:09:14,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-12T04:09:14,015 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:14,016 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:14,026 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:14,027 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80051 connected 2024-11-12T04:09:14,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-12T04:09:14,135 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80051, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:14,144 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:14,296 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:14,297 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=88 2024-11-12T04:09:14,297 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:09:14,298 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-12T04:09:14,298 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-12T04:09:14,299 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:14,299 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f8004f, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:14,299 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f8004f, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:14,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-12T04:09:14,410 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f8004f, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:14,410 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=88}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x362d02b4 to 127.0.0.1:50727 2024-11-12T04:09:14,411 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=88}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:14,435 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:14,435 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-12T04:09:14,436 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-11-12T04:09:14,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=88 2024-11-12T04:09:14,439 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:14,442 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=88, resume processing ppid=87 2024-11-12T04:09:14,442 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=[f1]},bandwidth=0,serial=false 2024-11-12T04:09:14,442 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=88, ppid=87, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 295 msec 2024-11-12T04:09:14,445 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=87, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 442 msec 2024-11-12T04:09:14,471 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x62b5ddd9 to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4d4b153e 2024-11-12T04:09:14,481 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:14,482 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:14,482 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@2f763105, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:14,483 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:14,493 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:14,494 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:14,494 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f80053 connected 2024-11-12T04:09:14,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=87 2024-11-12T04:09:14,628 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:09:14,629 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:14,630 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-11-12T04:09:14,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=89, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:09:14,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-11-12T04:09:14,656 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:14,657 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:14,668 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:14,669 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80054 connected 2024-11-12T04:09:14,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-11-12T04:09:14,777 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80054, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:14,801 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:14,867 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:09:14,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-11-12T04:09:14,954 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:14,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=90 2024-11-12T04:09:14,955 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:09:14,956 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-12T04:09:14,956 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-12T04:09:14,956 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:14,957 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f80053, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:14,957 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f80053, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:15,068 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f80053, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:15,068 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=90}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x62b5ddd9 to 127.0.0.1:50727 2024-11-12T04:09:15,069 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=90}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:15,074 ERROR [LeaseRenewer:jenkins@localhost:44407 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:44407,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:15,088 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:15,089 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-12T04:09:15,089 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-12T04:09:15,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=90 2024-11-12T04:09:15,091 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:15,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=90, resume processing ppid=89 2024-11-12T04:09:15,094 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=90, ppid=89, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 291 msec 2024-11-12T04:09:15,094 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testAppendPeerTableCFst1=null, testAppendPeerTableCFst3=[f1], testAppendPeerTableCFst2=null, testAppendPeerTableCFst5=null, testAppendPeerTableCFst4=[f1, f2], testAppendPeerTableCFst6=null},bandwidth=0,serial=false 2024-11-12T04:09:15,096 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=89, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 464 msec 2024-11-12T04:09:15,126 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x29ac3a68 to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2abe1a98 2024-11-12T04:09:15,135 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:15,136 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:15,137 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@71d2aa8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:15,138 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:15,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:09:15,152 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:15,152 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:15,152 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f80056 connected 2024-11-12T04:09:15,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=89 2024-11-12T04:09:15,258 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:09:15,258 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:15,259 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-12T04:09:15,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=91, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:15,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91 2024-11-12T04:09:15,262 DEBUG [PEWorker-5 {}] zookeeper.ZKUtil(111): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-12T04:09:15,272 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-12T04:09:15,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-12T04:09:15,273 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-12T04:09:15,273 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:15,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91 2024-11-12T04:09:15,426 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:15,426 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=92 2024-11-12T04:09:15,426 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-12T04:09:15,427 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-12T04:09:15,427 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-12T04:09:15,427 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:15,427 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f80056, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:15,427 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f80056, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:15,535 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f80056, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:15,535 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=92}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x29ac3a68 to 127.0.0.1:50727 2024-11-12T04:09:15,535 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=92}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:15,535 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-12T04:09:15,544 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=92}] zookeeper.ZKUtil(111): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1/354fa2a6db59%2C40009%2C1731384526494.1731384529497 2024-11-12T04:09:15,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1/354fa2a6db59%2C40009%2C1731384526494.1731384529497 2024-11-12T04:09:15,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1 2024-11-12T04:09:15,552 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1 2024-11-12T04:09:15,552 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=92}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-12T04:09:15,552 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-12T04:09:15,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=92 2024-11-12T04:09:15,556 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:15,560 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=92, resume processing ppid=91 2024-11-12T04:09:15,561 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=92, ppid=91, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 283 msec 2024-11-12T04:09:15,569 DEBUG [PEWorker-3 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-12T04:09:15,569 INFO [PEWorker-3 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-12T04:09:15,570 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=91, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 310 msec 2024-11-12T04:09:15,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=91 2024-11-12T04:09:15,578 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-12T04:09:15,579 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-12T04:09:15,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=93, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=93 2024-11-12T04:09:15,582 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:15,584 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=93, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:15,585 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=93, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-11-12T04:09:15,585 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:15,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=93 2024-11-12T04:09:15,688 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-12T04:09:15,689 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-12T04:09:15,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=94, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:15,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=94 2024-11-12T04:09:15,691 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:15,692 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=94, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:15,693 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=94, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-11-12T04:09:15,693 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:15,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=94 2024-11-12T04:09:15,798 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-12T04:09:15,810 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testAppendPeerTableCFs[0] Thread=218 (was 220), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=279 (was 284), ProcessCount=11 (was 11), AvailableMemoryMB=2222 (was 2425) 2024-11-12T04:09:15,818 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[0] Thread=218, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=279, ProcessCount=11, AvailableMemoryMB=2222 2024-11-12T04:09:15,820 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-12T04:09:15,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=95, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-12T04:09:15,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=95 2024-11-12T04:09:15,836 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:15,836 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:15,847 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:15,848 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80057 connected 2024-11-12T04:09:15,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=95 2024-11-12T04:09:15,960 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80057, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:15,986 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:16,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=95 2024-11-12T04:09:16,138 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:16,139 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=96 2024-11-12T04:09:16,139 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-12T04:09:16,153 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=96}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:16,166 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:16,210 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-12T04:09:16,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=96 2024-11-12T04:09:16,212 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:16,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=96, resume processing ppid=95 2024-11-12T04:09:16,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=96, ppid=95, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 226 msec 2024-11-12T04:09:16,215 INFO [PEWorker-1 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-12T04:09:16,217 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=95, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 395 msec 2024-11-12T04:09:16,242 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1b63645d to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@45e58683 2024-11-12T04:09:16,252 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:16,252 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:16,252 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@14895da7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:16,253 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:16,264 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:16,264 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:16,264 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f80059 connected 2024-11-12T04:09:16,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=95 2024-11-12T04:09:16,448 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-12T04:09:16,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-11-12T04:09:16,451 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3938): Client=jenkins//172.17.0.2 disable replication peer, id=1 2024-11-12T04:09:16,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=97, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure 2024-11-12T04:09:16,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=97 2024-11-12T04:09:16,465 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:16,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=97 2024-11-12T04:09:16,617 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:16,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=98 2024-11-12T04:09:16,618 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=DISABLE_PEER 2024-11-12T04:09:16,619 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-12T04:09:16,619 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-12T04:09:16,619 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:16,619 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f80059, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:16,620 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f80059, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:16,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=97 2024-11-12T04:09:16,905 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f80059, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:16,905 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=98}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1b63645d to 127.0.0.1:50727 2024-11-12T04:09:16,905 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=98}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:16,924 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:16,924 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-12T04:09:16,924 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-12T04:09:16,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=98 2024-11-12T04:09:16,927 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for DISABLE on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:16,928 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=98, resume processing ppid=97 2024-11-12T04:09:16,929 INFO [PEWorker-3 {}] replication.DisablePeerProcedure(67): Successfully disabled peer 1 2024-11-12T04:09:16,929 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=98, ppid=97, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 462 msec 2024-11-12T04:09:16,930 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=97, state=SUCCESS; org.apache.hadoop.hbase.master.replication.DisablePeerProcedure in 478 msec 2024-11-12T04:09:16,957 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x165cf3d9 to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@5091f337 2024-11-12T04:09:16,981 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:16,982 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:16,982 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@2e016fb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:16,983 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:16,993 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:16,993 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:16,993 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f8005b connected 2024-11-12T04:09:17,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=97 2024-11-12T04:09:17,078 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: DISABLE_REPLICATION_PEER, peerId: 1 completed 2024-11-12T04:09:17,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-11-12T04:09:17,080 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-12T04:09:17,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=99, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:17,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=99 2024-11-12T04:09:17,085 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-12T04:09:17,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-12T04:09:17,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-12T04:09:17,098 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-12T04:09:17,100 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:17,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=99 2024-11-12T04:09:17,253 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:17,253 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=100 2024-11-12T04:09:17,253 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-12T04:09:17,254 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-12T04:09:17,254 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-12T04:09:17,254 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:17,254 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f8005b, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:17,254 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f8005b, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:17,364 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f8005b, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:17,364 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=100}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x165cf3d9 to 127.0.0.1:50727 2024-11-12T04:09:17,365 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=100}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:17,365 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-12T04:09:17,373 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=100}] zookeeper.ZKUtil(111): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1/354fa2a6db59%2C40009%2C1731384526494.1731384529497 2024-11-12T04:09:17,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1/354fa2a6db59%2C40009%2C1731384526494.1731384529497 2024-11-12T04:09:17,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1 2024-11-12T04:09:17,385 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1 2024-11-12T04:09:17,385 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=100}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-12T04:09:17,385 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-12T04:09:17,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=100 2024-11-12T04:09:17,388 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:17,390 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=100, resume processing ppid=99 2024-11-12T04:09:17,390 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=100, ppid=99, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 289 msec 2024-11-12T04:09:17,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=99 2024-11-12T04:09:17,402 DEBUG [PEWorker-2 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-12T04:09:17,402 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-12T04:09:17,403 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=99, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 322 msec 2024-11-12T04:09:17,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=99 2024-11-12T04:09:17,708 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-12T04:09:17,708 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-12T04:09:17,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=101, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:17,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=101 2024-11-12T04:09:17,710 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:17,711 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=101, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:17,711 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=101, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-11-12T04:09:17,711 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:17,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=101 2024-11-12T04:09:17,818 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-12T04:09:17,818 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-12T04:09:17,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=102, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:17,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=102 2024-11-12T04:09:17,820 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:17,822 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=102, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:17,823 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=102, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-11-12T04:09:17,823 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:17,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=102 2024-11-12T04:09:17,928 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-12T04:09:17,942 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testEnableDisablePeer[0] Thread=218 (was 218), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=279 (was 279), ProcessCount=11 (was 11), AvailableMemoryMB=2172 (was 2222) 2024-11-12T04:09:17,955 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[0] Thread=218, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=279, ProcessCount=11, AvailableMemoryMB=2171 2024-11-12T04:09:17,956 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=dummy_peer 2024-11-12T04:09:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=103, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:17,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=103 2024-11-12T04:09:17,958 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer dummy_peer, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:17,960 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=103, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:17,961 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=103, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: dummy_peer; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-11-12T04:09:17,961 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=103 2024-11-12T04:09:18,068 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: dummy_peer failed with dummy_peer 2024-11-12T04:09:18,069 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-12T04:09:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=104, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=104 2024-11-12T04:09:18,071 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:18,073 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=104, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:18,074 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=104, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-11-12T04:09:18,074 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:18,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=104 2024-11-12T04:09:18,179 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-12T04:09:18,180 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-12T04:09:18,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=105, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:18,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=105 2024-11-12T04:09:18,185 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:18,187 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=105, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:18,188 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=105, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=7 msec 2024-11-12T04:09:18,188 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:18,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=105 2024-11-12T04:09:18,289 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-12T04:09:18,308 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerNotFoundException[0] Thread=218 (was 218), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=279 (was 279), ProcessCount=11 (was 11), AvailableMemoryMB=2165 (was 2171) 2024-11-12T04:09:18,316 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerConfig[0] Thread=218, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=279, ProcessCount=11, AvailableMemoryMB=2164 2024-11-12T04:09:18,317 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-12T04:09:18,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=106, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-12T04:09:18,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-11-12T04:09:18,336 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:18,336 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:18,347 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:18,348 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8005c connected 2024-11-12T04:09:18,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-11-12T04:09:18,456 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8005c, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:18,478 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:18,630 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:18,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=107 2024-11-12T04:09:18,632 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-12T04:09:18,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-11-12T04:09:18,652 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=107}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:18,665 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:18,735 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=107}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=107 2024-11-12T04:09:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=107 2024-11-12T04:09:18,739 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:18,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=107, resume processing ppid=106 2024-11-12T04:09:18,743 INFO [PEWorker-5 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-12T04:09:18,743 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=107, ppid=106, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 261 msec 2024-11-12T04:09:18,746 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=106, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 425 msec 2024-11-12T04:09:18,773 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x10d6a0df to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@4162ffc0 2024-11-12T04:09:18,785 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:18,786 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:18,786 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@4b60f78b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:18,789 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:18,797 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:18,798 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:18,798 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f8005e connected 2024-11-12T04:09:18,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=106 2024-11-12T04:09:18,948 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-12T04:09:18,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3971): Client=jenkins//172.17.0.2 list replication peers, regex=null 2024-11-12T04:09:18,949 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-12T04:09:18,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=108, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:18,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-11-12T04:09:18,954 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-12T04:09:18,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-12T04:09:18,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-12T04:09:18,964 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-12T04:09:18,965 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=109, ppid=108, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:19,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-11-12T04:09:19,117 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:19,117 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=109 2024-11-12T04:09:19,117 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-12T04:09:19,118 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-12T04:09:19,118 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-12T04:09:19,118 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:19,118 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f8005e, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:19,118 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f8005e, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:19,227 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f8005e, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:19,227 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=109}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x10d6a0df to 127.0.0.1:50727 2024-11-12T04:09:19,227 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=109}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:19,227 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-12T04:09:19,235 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=109}] zookeeper.ZKUtil(111): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1/354fa2a6db59%2C40009%2C1731384526494.1731384529497 2024-11-12T04:09:19,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1/354fa2a6db59%2C40009%2C1731384526494.1731384529497 2024-11-12T04:09:19,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1 2024-11-12T04:09:19,243 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1 2024-11-12T04:09:19,244 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=109}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-12T04:09:19,244 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=109}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=109 2024-11-12T04:09:19,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=109 2024-11-12T04:09:19,248 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:19,251 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=109, resume processing ppid=108 2024-11-12T04:09:19,251 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=109, ppid=108, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 283 msec 2024-11-12T04:09:19,260 DEBUG [PEWorker-1 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-12T04:09:19,261 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-12T04:09:19,263 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=108, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 312 msec 2024-11-12T04:09:19,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=108 2024-11-12T04:09:19,268 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-12T04:09:19,268 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-12T04:09:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=110, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:19,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-11-12T04:09:19,269 WARN [PEWorker-5 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:19,270 ERROR [PEWorker-5 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=110, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:19,271 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=110, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=3 msec 2024-11-12T04:09:19,271 WARN [PEWorker-5 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:19,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=110 2024-11-12T04:09:19,380 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-12T04:09:19,381 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-12T04:09:19,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=111, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:19,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=111 2024-11-12T04:09:19,384 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:19,387 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=111, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:19,389 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=111, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=6 msec 2024-11-12T04:09:19,389 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:19,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=111 2024-11-12T04:09:19,489 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-12T04:09:19,504 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerConfig[0] Thread=218 (was 218), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=281 (was 279) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2123 (was 2164) 2024-11-12T04:09:19,514 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[0] Thread=218, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=281, ProcessCount=11, AvailableMemoryMB=2122 2024-11-12T04:09:19,514 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:44407 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:44407,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:19,519 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-12T04:09:19,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-12T04:09:19,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] ipc.CallRunner(138): callId: 354 service: MasterService methodName: AddReplicationPeer size: 80 connection: 172.17.0.2:43442 deadline: 1731384619519, exception=java.io.IOException: Replication peer modification disabled 2024-11-12T04:09:19,521 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(168): Call to master failed, tries = 1, maxAttempts = 2, timeout = 120000 ms, time elapsed = 1 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:245) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:431) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 33 more 2024-11-12T04:09:19,628 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-12T04:09:19,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] ipc.MetricsHBaseServer(152): Unknown exception type java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) ~[hbase-protocol-shaded-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) ~[classes/:2.7.0-SNAPSHOT] 2024-11-12T04:09:19,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] ipc.CallRunner(138): callId: 355 service: MasterService methodName: AddReplicationPeer size: 80 connection: 172.17.0.2:43442 deadline: 1731384619627, exception=java.io.IOException: Replication peer modification disabled 2024-11-12T04:09:19,629 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(168): Call to master failed, tries = 2, maxAttempts = 2, timeout = 120000 ms, time elapsed = 110 ms java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance0(Native Method) ~[?:?] at jdk.internal.reflect.NativeConstructorAccessorImpl.newInstance(NativeConstructorAccessorImpl.java:77) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:245) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.AsyncMasterRequestRpcRetryingCaller.lambda$doCall$4(AsyncMasterRequestRpcRetryingCaller.java:76) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.RawAsyncHBaseAdmin$1.run(RawAsyncHBaseAdmin.java:431) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.access$100(AbstractRpcClient.java:94) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:116) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:131) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 33 more 2024-11-12T04:09:19,631 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 1 failed with Failed after attempts=2, exceptions: 2024-11-12T04:09:19.522Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) 2024-11-12T04:09:19.630Z, java.io.IOException: java.io.IOException: Replication peer modification disabled at org.apache.hadoop.hbase.master.HMaster.executePeerProcedure(HMaster.java:3909) at org.apache.hadoop.hbase.master.HMaster.addReplicationPeer(HMaster.java:3921) at org.apache.hadoop.hbase.master.MasterRpcServices.addReplicationPeer(MasterRpcServices.java:2115) at org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$2.callBlockingMethod(MasterProtos.java) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:443) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:105) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) 2024-11-12T04:09:19,634 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-12T04:09:19,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=112, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:19,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-11-12T04:09:19,637 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:19,639 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=112, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:19,640 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=112, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-11-12T04:09:19,641 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:19,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=112 2024-11-12T04:09:19,751 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-12T04:09:19,752 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-12T04:09:19,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=113, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:19,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=113 2024-11-12T04:09:19,757 WARN [PEWorker-3 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:19,760 ERROR [PEWorker-3 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=113, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:19,762 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=113, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=8 msec 2024-11-12T04:09:19,762 WARN [PEWorker-3 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:19,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=113 2024-11-12T04:09:19,870 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-12T04:09:19,888 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testReplicationPeerModificationSwitch[0] Thread=217 (was 218), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=281 (was 281), ProcessCount=11 (was 11), AvailableMemoryMB=2117 (was 2122) 2024-11-12T04:09:19,898 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[0] Thread=217, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=281, ProcessCount=11, AvailableMemoryMB=2116 2024-11-12T04:09:19,899 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-12T04:09:19,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=114, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-12T04:09:19,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-11-12T04:09:19,916 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:19,917 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:19,926 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:19,927 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8005f connected 2024-11-12T04:09:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-11-12T04:09:20,035 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8005f, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:20,058 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:20,211 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:20,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=115 2024-11-12T04:09:20,212 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-12T04:09:20,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-11-12T04:09:20,227 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=115}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:20,241 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:20,285 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=115}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=115 2024-11-12T04:09:20,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=115 2024-11-12T04:09:20,287 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:20,289 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=115, resume processing ppid=114 2024-11-12T04:09:20,289 INFO [PEWorker-4 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-12T04:09:20,289 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=115, ppid=114, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 229 msec 2024-11-12T04:09:20,291 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=114, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 390 msec 2024-11-12T04:09:20,318 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x27b2dde6 to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@154728ed 2024-11-12T04:09:20,327 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:20,327 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:20,327 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@cebb538, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:20,328 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:20,339 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:20,340 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:20,340 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f80061 connected 2024-11-12T04:09:20,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=114 2024-11-12T04:09:20,528 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-12T04:09:20,528 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:20,529 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-12T04:09:20,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=116, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:20,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116 2024-11-12T04:09:20,533 DEBUG [PEWorker-3 {}] zookeeper.ZKUtil(111): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-12T04:09:20,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-12T04:09:20,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-12T04:09:20,543 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-12T04:09:20,544 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:20,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116 2024-11-12T04:09:20,695 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:20,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=117 2024-11-12T04:09:20,696 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-12T04:09:20,696 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-12T04:09:20,696 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-12T04:09:20,696 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:20,696 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f80061, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:20,697 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f80061, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:20,807 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f80061, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:20,807 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=117}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x27b2dde6 to 127.0.0.1:50727 2024-11-12T04:09:20,807 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=117}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:20,807 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-12T04:09:20,818 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=117}] zookeeper.ZKUtil(111): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1/354fa2a6db59%2C40009%2C1731384526494.1731384529497 2024-11-12T04:09:20,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1/354fa2a6db59%2C40009%2C1731384526494.1731384529497 2024-11-12T04:09:20,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1 2024-11-12T04:09:20,827 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1 2024-11-12T04:09:20,827 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=117}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-12T04:09:20,827 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=117}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=117 2024-11-12T04:09:20,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=117 2024-11-12T04:09:20,831 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:20,835 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=117, resume processing ppid=116 2024-11-12T04:09:20,835 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=117, ppid=116, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 287 msec 2024-11-12T04:09:20,844 DEBUG [PEWorker-1 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-12T04:09:20,844 INFO [PEWorker-1 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-12T04:09:20,846 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=116, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 315 msec 2024-11-12T04:09:20,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=116 2024-11-12T04:09:20,848 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-12T04:09:20,848 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-12T04:09:20,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=118, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:20,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=118 2024-11-12T04:09:20,850 WARN [PEWorker-4 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:20,852 ERROR [PEWorker-4 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=118, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:20,853 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=118, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-11-12T04:09:20,853 WARN [PEWorker-4 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:20,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=118 2024-11-12T04:09:20,958 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-12T04:09:20,976 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testClusterKeyWithTrailingSpace[0] Thread=216 (was 217), OpenFileDescriptor=444 (was 444), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=281 (was 281), ProcessCount=11 (was 11), AvailableMemoryMB=2026 (was 2116) 2024-11-12T04:09:20,985 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testPeerBandwidth[0] Thread=216, OpenFileDescriptor=444, MaxFileDescriptor=1048576, SystemLoadAverage=281, ProcessCount=11, AvailableMemoryMB=2025 2024-11-12T04:09:20,986 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-12T04:09:20,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=119, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-12T04:09:20,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119 2024-11-12T04:09:21,011 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:21,012 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:21,023 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:21,023 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80062 connected 2024-11-12T04:09:21,056 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T04:09:21,086 DEBUG [BootstrapNodeManager {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T04:09:21,088 INFO [RS-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60768, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T04:09:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119 2024-11-12T04:09:21,131 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80062, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:21,152 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:21,304 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:21,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=120 2024-11-12T04:09:21,305 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-12T04:09:21,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119 2024-11-12T04:09:21,326 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=120}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:21,341 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:21,438 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-12T04:09:21,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=120 2024-11-12T04:09:21,440 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:21,441 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=120, resume processing ppid=119 2024-11-12T04:09:21,441 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=120, ppid=119, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 288 msec 2024-11-12T04:09:21,441 INFO [PEWorker-1 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-12T04:09:21,442 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=119, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 454 msec 2024-11-12T04:09:21,464 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x77f0fd06 to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@23991381 2024-11-12T04:09:21,473 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:21,473 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:21,474 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@782c9932, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:21,475 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:21,485 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:21,485 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:21,485 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f80064 connected 2024-11-12T04:09:21,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=119 2024-11-12T04:09:21,618 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-12T04:09:21,619 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:21,619 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-11-12T04:09:21,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=121, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:09:21,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=121 2024-11-12T04:09:21,635 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:21,636 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:21,670 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:21,670 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80065 connected 2024-11-12T04:09:21,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=121 2024-11-12T04:09:21,781 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80065, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:21,794 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:21,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=121 2024-11-12T04:09:21,947 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:21,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=122 2024-11-12T04:09:21,948 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:09:21,949 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-12T04:09:21,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=122 2024-11-12T04:09:21,951 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:21,953 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=122, resume processing ppid=121 2024-11-12T04:09:21,953 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=2097152,serial=false 2024-11-12T04:09:21,953 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=122, ppid=121, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 157 msec 2024-11-12T04:09:21,954 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=121, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 334 msec 2024-11-12T04:09:22,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=121 2024-11-12T04:09:22,248 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:09:22,249 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:22,249 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-12T04:09:22,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=123, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:22,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=123 2024-11-12T04:09:22,253 DEBUG [PEWorker-1 {}] zookeeper.ZKUtil(111): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-12T04:09:22,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-12T04:09:22,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-12T04:09:22,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-12T04:09:22,261 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=123 2024-11-12T04:09:22,413 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:22,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=124 2024-11-12T04:09:22,413 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-12T04:09:22,413 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-12T04:09:22,413 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-12T04:09:22,414 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:22,414 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f80064, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:22,414 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f80064, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:22,523 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f80064, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:22,523 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=124}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x77f0fd06 to 127.0.0.1:50727 2024-11-12T04:09:22,523 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=124}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:22,523 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-12T04:09:22,531 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=124}] zookeeper.ZKUtil(111): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1/354fa2a6db59%2C40009%2C1731384526494.1731384529497 2024-11-12T04:09:22,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1/354fa2a6db59%2C40009%2C1731384526494.1731384529497 2024-11-12T04:09:22,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1 2024-11-12T04:09:22,539 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1 2024-11-12T04:09:22,539 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=124}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-12T04:09:22,540 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-12T04:09:22,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=124 2024-11-12T04:09:22,542 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:22,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=124, resume processing ppid=123 2024-11-12T04:09:22,544 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=124, ppid=123, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 281 msec 2024-11-12T04:09:22,552 DEBUG [PEWorker-5 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-12T04:09:22,552 INFO [PEWorker-5 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-12T04:09:22,554 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=123, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 303 msec 2024-11-12T04:09:22,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=123 2024-11-12T04:09:22,568 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-12T04:09:22,568 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-12T04:09:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=125, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:22,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=125 2024-11-12T04:09:22,571 WARN [PEWorker-2 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 1, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:22,572 ERROR [PEWorker-2 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=125, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:22,572 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=125, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 1; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-11-12T04:09:22,572 WARN [PEWorker-2 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=125 2024-11-12T04:09:22,679 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 1 failed with 1 2024-11-12T04:09:22,680 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-12T04:09:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=126, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:22,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=126 2024-11-12T04:09:22,683 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:22,685 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=126, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:22,686 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=126, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=5 msec 2024-11-12T04:09:22,686 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:22,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=126 2024-11-12T04:09:22,788 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-12T04:09:22,804 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testPeerBandwidth[0] Thread=217 (was 216) - Thread LEAK? -, OpenFileDescriptor=446 (was 444) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=281 (was 281), ProcessCount=11 (was 11), AvailableMemoryMB=1941 (was 2025) 2024-11-12T04:09:22,813 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[0] Thread=217, OpenFileDescriptor=446, MaxFileDescriptor=1048576, SystemLoadAverage=281, ProcessCount=11, AvailableMemoryMB=1940 2024-11-12T04:09:22,815 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-12T04:09:22,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=127, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-12T04:09:22,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=127 2024-11-12T04:09:22,836 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:22,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=127 2024-11-12T04:09:22,989 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:22,989 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=128 2024-11-12T04:09:22,990 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=128}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-12T04:09:23,020 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=128}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:23,068 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-12T04:09:23,068 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(587): peerId=1, queueId=1 (queues=1) is replicating from cluster=f839b423-66d7-495f-b21c-29c55cacf7ac to cluster=f839b423-66d7-495f-b21c-29c55cacf7ac 2024-11-12T04:09:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=128 2024-11-12T04:09:23,069 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(343): peerId=1, starting shipping worker for walGroupId=354fa2a6db59%2C40009%2C1731384526494 2024-11-12T04:09:23,070 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:23,072 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=128, resume processing ppid=127 2024-11-12T04:09:23,072 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSourceWALReader(112): peerClusterZnode=1, ReplicationSourceWALReaderThread : 1 inited, replicationBatchSizeCapacity=67108864, replicationBatchCountCapacity=25000, replicationBatchQueueCapacity=1 2024-11-12T04:09:23,072 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=128, ppid=127, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 234 msec 2024-11-12T04:09:23,072 INFO [PEWorker-2 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.VerifyWALEntriesReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-12T04:09:23,073 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=127, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 257 msec 2024-11-12T04:09:23,074 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1.replicationSource.shipper354fa2a6db59%2C40009%2C1731384526494,1 {}] regionserver.ReplicationSourceShipper(98): Running ReplicationSourceShipper Thread for wal group: 354fa2a6db59%2C40009%2C1731384526494 2024-11-12T04:09:23,074 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1.replicationSource.wal-reader.354fa2a6db59%2C40009%2C1731384526494,1 {}] regionserver.WALEntryStream(260): Creating new reader hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/WALs/354fa2a6db59,40009,1731384526494/354fa2a6db59%2C40009%2C1731384526494.1731384529497, startPosition=0, beingWritten=true 2024-11-12T04:09:23,134 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1.replicationSource.wal-reader.354fa2a6db59%2C40009%2C1731384526494,1 {}] regionserver.ReplicationSourceWALReader(181): Read 0 WAL entries eligible for replication 2024-11-12T04:09:23,134 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1.replicationSource.wal-reader.354fa2a6db59%2C40009%2C1731384526494,1 {}] regionserver.WALEntryStream(223): Reset reader hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/WALs/354fa2a6db59,40009,1731384526494/354fa2a6db59%2C40009%2C1731384526494.1731384529497 to pos 589, reset compression=false 2024-11-12T04:09:23,134 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1.replicationSource.shipper354fa2a6db59%2C40009%2C1731384526494,1 {}] regionserver.ReplicationSourceShipper(110): Shipper from source 1 got entry batch from reader: WALEntryBatch [walEntries=[], lastWalPath=hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/WALs/354fa2a6db59,40009,1731384526494/354fa2a6db59%2C40009%2C1731384526494.1731384529497, lastWalPosition=589, nbRowKeys=0, nbHFiles=0, heapSize=0, lastSeqIds={}, endOfFile=false,usedBufferSize=0] 2024-11-12T04:09:23,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=127 2024-11-12T04:09:23,137 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-12T04:09:23,138 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=2, config=clusterKey=,replicationEndpointImpl=org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-12T04:09:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=129, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-12T04:09:23,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=129 2024-11-12T04:09:23,140 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.AddPeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] ... 13 more 2024-11-12T04:09:23,141 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=129, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:395) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more Caused by: java.io.IOException: Cluster key passed is invalid, the format should be:hbase.zookeeper.quorum:hbase.zookeeper.property.clientPort:zookeeper.znode.parent at org.apache.hadoop.hbase.zookeeper.ZKConfig.transformClusterKey(ZKConfig.java:228) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKConfig.validateClusterKey(ZKConfig.java:178) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkClusterKey(ReplicationPeerManager.java:393) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerConfig(ReplicationPeerManager.java:298) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preAddPeer(ReplicationPeerManager.java:119) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.AddPeerProcedure.prePeerModification(AddPeerProcedure.java:90) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:23,142 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=129, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.DoNotRetryIOException via master-add-peer:org.apache.hadoop.hbase.DoNotRetryIOException: Invalid cluster key: ; org.apache.hadoop.hbase.master.replication.AddPeerProcedure exec-time=3 msec 2024-11-12T04:09:23,142 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:23,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=129 2024-11-12T04:09:23,248 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: ADD_REPLICATION_PEER, peerId: 2 failed with Invalid cluster key: 2024-11-12T04:09:23,248 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-12T04:09:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=130, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:23,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-12T04:09:23,251 DEBUG [PEWorker-4 {}] zookeeper.ZKUtil(111): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-12T04:09:23,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-12T04:09:23,259 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-12T04:09:23,260 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-12T04:09:23,260 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=131, ppid=130, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:23,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-12T04:09:23,412 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:23,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=131 2024-11-12T04:09:23,413 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-12T04:09:23,413 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-12T04:09:23,413 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-12T04:09:23,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-12T04:09:23,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-12T04:09:24,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-12T04:09:24,413 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1.replicationSource.wal-reader.354fa2a6db59%2C40009%2C1731384526494,1 {}] util.Threads(127): sleep interrupted java.lang.InterruptedException: sleep interrupted at java.lang.Thread.sleep(Native Method) ~[?:?] at org.apache.hadoop.hbase.util.Threads.sleep(Threads.java:125) ~[hbase-common-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.sleep(ReplicationSourceWALReader.java:131) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.run(ReplicationSourceWALReader.java:166) ~[classes/:?] 2024-11-12T04:09:24,413 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1.replicationSource.shipper354fa2a6db59%2C40009%2C1731384526494,1 {}] regionserver.ReplicationSourceShipper(123): Interrupted while waiting for next replication entry batch java.lang.InterruptedException: null at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1681) ~[?:?] at java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) ~[?:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceWALReader.poll(ReplicationSourceWALReader.java:313) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSourceShipper.run(ReplicationSourceShipper.java:109) ~[classes/:?] 2024-11-12T04:09:25,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:09:25,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-12T04:09:25,414 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.ReplicationSource(711): peerId=1, ReplicationSourceWorker RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1.replicationSource.shipper354fa2a6db59%2C40009%2C1731384526494,1 terminated 2024-11-12T04:09:25,414 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-12T04:09:25,415 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=131}] zookeeper.ZKUtil(111): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1/354fa2a6db59%2C40009%2C1731384526494.1731384529497 2024-11-12T04:09:25,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1/354fa2a6db59%2C40009%2C1731384526494.1731384529497 2024-11-12T04:09:25,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1 2024-11-12T04:09:25,426 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1 2024-11-12T04:09:25,426 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=131}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-12T04:09:25,426 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=131}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=131 2024-11-12T04:09:25,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=131 2024-11-12T04:09:25,429 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:25,431 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=131, resume processing ppid=130 2024-11-12T04:09:25,431 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=131, ppid=130, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 2.1690 sec 2024-11-12T04:09:25,444 DEBUG [PEWorker-2 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-12T04:09:25,444 INFO [PEWorker-2 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-12T04:09:25,447 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=130, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 2.1960 sec 2024-11-12T04:09:26,310 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T04:09:27,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=130 2024-11-12T04:09:27,428 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-12T04:09:27,428 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=2 2024-11-12T04:09:27,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=132, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:27,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-11-12T04:09:27,430 WARN [PEWorker-1 {}] replication.ModifyPeerProcedure(282): org.apache.hadoop.hbase.master.replication.RemovePeerProcedure failed to call pre CP hook or the pre check is failed for peer 2, mark the procedure as failure and give up org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] 2024-11-12T04:09:27,432 ERROR [PEWorker-1 {}] procedure2.ProcedureExecutor(1656): Root Procedure pid=132, state=FAILED, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure does not support rollback but the execution failed and try to rollback, code bug? org.apache.hadoop.hbase.procedure2.RemoteProcedureException: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.procedure2.Procedure.setFailure(Procedure.java:768) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:284) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:54) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.StateMachineProcedure.execute(StateMachineProcedure.java:188) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.Procedure.doExecute(Procedure.java:944) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.execProcedure(ProcedureExecutor.java:1784) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.executeProcedure(ProcedureExecutor.java:1462) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor.access$1000(ProcedureExecutor.java:77) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.runProcedure(ProcedureExecutor.java:2106) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2133) ~[hbase-procedure-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] Caused by: org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2 at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.checkPeerExists(ReplicationPeerManager.java:133) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ReplicationPeerManager.preRemovePeer(ReplicationPeerManager.java:139) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.RemovePeerProcedure.prePeerModification(RemovePeerProcedure.java:61) ~[classes/:?] at org.apache.hadoop.hbase.master.replication.ModifyPeerProcedure.executeFromState(ModifyPeerProcedure.java:280) ~[classes/:?] ... 9 more 2024-11-12T04:09:27,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1666): Rolled back pid=132, state=ROLLEDBACK, locked=true, exception=org.apache.hadoop.hbase.ReplicationPeerNotFoundException via master-remove-peer:org.apache.hadoop.hbase.ReplicationPeerNotFoundException: 2; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure exec-time=4 msec 2024-11-12T04:09:27,434 WARN [PEWorker-1 {}] procedure2.ProcedureExecutor(2020): Usually this should not happen, we will release the lock before if the procedure is finished, even if the holdLock is true, arrive here means we have some holes where we do not release the lock. And the releaseLock below may fail since the procedure may have already been deleted from the procedure store. 2024-11-12T04:09:27,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=132 2024-11-12T04:09:27,539 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2976): Operation: REMOVE_REPLICATION_PEER, peerId: 2 failed with 2 2024-11-12T04:09:27,554 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: client.TestAsyncReplicationAdminApi#testSetReplicationEndpoint[0] Thread=219 (was 217) - Thread LEAK? -, OpenFileDescriptor=451 (was 446) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=258 (was 281), ProcessCount=11 (was 11), AvailableMemoryMB=1841 (was 1940) 2024-11-12T04:09:27,565 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: client.TestAsyncReplicationAdminApi#testRemovePeerTableCFs[1] Thread=219, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=258, ProcessCount=11, AvailableMemoryMB=1841 2024-11-12T04:09:27,567 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3919): Client=jenkins//172.17.0.2 creating replication peer, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false, state=ENABLED 2024-11-12T04:09:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=133, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.AddPeerProcedure 2024-11-12T04:09:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=133 2024-11-12T04:09:27,585 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:27,585 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:27,597 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:27,598 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80066 connected 2024-11-12T04:09:27,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=133 2024-11-12T04:09:27,706 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80066, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:27,727 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:27,879 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:27,880 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=134 2024-11-12T04:09:27,880 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=ADD_PEER 2024-11-12T04:09:27,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=133 2024-11-12T04:09:27,896 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=134}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:27,908 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:27,951 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-12T04:09:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=134 2024-11-12T04:09:27,955 INFO [PEWorker-5 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for ADD on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:27,957 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=134, resume processing ppid=133 2024-11-12T04:09:27,958 INFO [PEWorker-2 {}] replication.AddPeerProcedure(102): Successfully added ENABLED peer 1, config clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=true,bandwidth=0,serial=false 2024-11-12T04:09:27,958 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=134, ppid=133, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 228 msec 2024-11-12T04:09:27,960 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=133, state=SUCCESS; org.apache.hadoop.hbase.master.replication.AddPeerProcedure in 391 msec 2024-11-12T04:09:27,984 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x06dd90b2 to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@1bbf7e09 2024-11-12T04:09:27,993 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:27,993 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:27,994 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@53b74770, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:27,994 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:28,001 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:28,001 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:28,001 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f80068 connected 2024-11-12T04:09:28,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=133 2024-11-12T04:09:28,198 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: ADD_REPLICATION_PEER, peerId: 1 completed 2024-11-12T04:09:28,199 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-12T04:09:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=135, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:09:28,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=135 2024-11-12T04:09:28,216 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:28,216 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:28,247 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:28,248 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80069 connected 2024-11-12T04:09:28,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=135 2024-11-12T04:09:28,360 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80069, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:28,370 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:28,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=135 2024-11-12T04:09:28,522 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:28,523 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=136 2024-11-12T04:09:28,523 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:09:28,525 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-12T04:09:28,525 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-12T04:09:28,525 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:28,525 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f80068, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:28,526 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f80068, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:28,635 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f80068, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:28,635 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=136}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x06dd90b2 to 127.0.0.1:50727 2024-11-12T04:09:28,635 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=136}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:28,648 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:28,649 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-12T04:09:28,649 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-12T04:09:28,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=136 2024-11-12T04:09:28,651 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:28,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=136, resume processing ppid=135 2024-11-12T04:09:28,653 INFO [PEWorker-5 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-12T04:09:28,653 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=136, ppid=135, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 281 msec 2024-11-12T04:09:28,654 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1480): Finished pid=135, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 454 msec 2024-11-12T04:09:28,676 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x09a195a0 to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@29cce228 2024-11-12T04:09:28,685 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:28,685 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:28,685 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@1de835a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:28,686 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:28,697 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:28,698 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:28,698 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f8006b connected 2024-11-12T04:09:28,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=135 2024-11-12T04:09:28,828 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:09:28,829 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:28,830 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:28,831 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:28,833 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-11-12T04:09:28,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=137, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:09:28,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=137 2024-11-12T04:09:28,850 INFO [PEWorker-2 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:28,851 INFO [PEWorker-2 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:28,859 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:28,860 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8006c connected 2024-11-12T04:09:28,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=137 2024-11-12T04:09:28,968 DEBUG [PEWorker-2-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8006c, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:28,978 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:29,130 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:29,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=138 2024-11-12T04:09:29,131 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:09:29,133 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-12T04:09:29,133 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-12T04:09:29,133 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:29,133 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f8006b, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:29,133 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f8006b, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:29,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=137 2024-11-12T04:09:29,243 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f8006b, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:29,243 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=138}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x09a195a0 to 127.0.0.1:50727 2024-11-12T04:09:29,243 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=138}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:29,261 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:29,262 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-12T04:09:29,262 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-12T04:09:29,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=138 2024-11-12T04:09:29,264 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:29,266 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=138, resume processing ppid=137 2024-11-12T04:09:29,266 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1], testRemovePeerTableCFst1=null},bandwidth=0,serial=false 2024-11-12T04:09:29,266 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=138, ppid=137, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 287 msec 2024-11-12T04:09:29,267 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=137, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 433 msec 2024-11-12T04:09:29,292 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x67067195 to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3ac78542 2024-11-12T04:09:29,301 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:29,302 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:29,302 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@6a147816, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:29,303 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:29,309 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:29,310 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:29,310 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f8006e connected 2024-11-12T04:09:29,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=137 2024-11-12T04:09:29,469 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:09:29,470 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:29,471 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:29,472 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:29,474 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:29,475 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-11-12T04:09:29,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=139, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:09:29,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139 2024-11-12T04:09:29,496 INFO [PEWorker-5 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:29,496 INFO [PEWorker-5 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:29,506 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:29,506 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8006f connected 2024-11-12T04:09:29,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139 2024-11-12T04:09:29,614 DEBUG [PEWorker-5-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f8006f, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:29,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:29,782 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:29,782 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=140 2024-11-12T04:09:29,782 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:09:29,783 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-12T04:09:29,783 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-12T04:09:29,783 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:29,784 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f8006e, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:29,784 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f8006e, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:29,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139 2024-11-12T04:09:29,918 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f8006e, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:29,918 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=140}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x67067195 to 127.0.0.1:50727 2024-11-12T04:09:29,918 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=140}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:29,937 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:29,938 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-12T04:09:29,938 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=140}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=140 2024-11-12T04:09:29,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=140 2024-11-12T04:09:29,940 INFO [PEWorker-1 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:29,941 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=140, resume processing ppid=139 2024-11-12T04:09:29,942 INFO [PEWorker-3 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst2=[cf1]},bandwidth=0,serial=false 2024-11-12T04:09:29,942 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=140, ppid=139, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 312 msec 2024-11-12T04:09:29,943 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=139, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 467 msec 2024-11-12T04:09:29,964 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x78b5946a to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@2764a9a6 2024-11-12T04:09:29,976 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:29,977 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:29,977 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@fc82ba3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:29,978 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:29,984 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:29,985 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:29,985 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f80071 connected 2024-11-12T04:09:30,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=139 2024-11-12T04:09:30,108 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:09:30,109 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:30,109 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:30,110 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:30,110 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-12T04:09:30,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=141, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:09:30,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=141 2024-11-12T04:09:30,126 INFO [PEWorker-4 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:30,126 INFO [PEWorker-4 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:30,134 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:30,135 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80072 connected 2024-11-12T04:09:30,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=141 2024-11-12T04:09:30,243 DEBUG [PEWorker-4-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80072, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:30,252 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:30,403 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:30,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=142 2024-11-12T04:09:30,404 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:09:30,405 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-12T04:09:30,405 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-12T04:09:30,405 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:30,405 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f80071, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:30,406 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f80071, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:30,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=141 2024-11-12T04:09:30,515 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f80071, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:30,515 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=142}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x78b5946a to 127.0.0.1:50727 2024-11-12T04:09:30,515 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=142}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:30,538 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:30,538 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-12T04:09:30,539 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=142}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=142 2024-11-12T04:09:30,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=142 2024-11-12T04:09:30,540 INFO [PEWorker-2 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:30,542 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=142, resume processing ppid=141 2024-11-12T04:09:30,542 INFO [PEWorker-1 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-12T04:09:30,542 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=142, ppid=141, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 288 msec 2024-11-12T04:09:30,566 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1480): Finished pid=141, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 431 msec 2024-11-12T04:09:30,587 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7a79c9fc to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@7ac16395 2024-11-12T04:09:30,598 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:30,598 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:30,598 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@5dc107ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:30,599 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:30,609 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:30,610 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:30,610 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f80074 connected 2024-11-12T04:09:30,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=141 2024-11-12T04:09:30,738 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:09:30,739 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:30,740 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:30,742 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-11-12T04:09:30,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=143, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:09:30,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143 2024-11-12T04:09:30,758 INFO [PEWorker-3 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:30,758 INFO [PEWorker-3 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:30,767 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:30,768 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80075 connected 2024-11-12T04:09:30,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143 2024-11-12T04:09:30,876 DEBUG [PEWorker-3-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id-0x1012e84e2f80075, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:30,891 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:31,045 DEBUG [RSProcedureDispatcher-pool-1 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:31,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=144 2024-11-12T04:09:31,047 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:09:31,050 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-12T04:09:31,050 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-12T04:09:31,050 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:31,051 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f80074, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:31,051 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f80074, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143 2024-11-12T04:09:31,160 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f80074, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:31,160 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=144}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7a79c9fc to 127.0.0.1:50727 2024-11-12T04:09:31,160 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=144}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:31,173 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:31,173 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-12T04:09:31,174 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=144}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=144 2024-11-12T04:09:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=144 2024-11-12T04:09:31,176 INFO [PEWorker-4 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:31,177 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=144, resume processing ppid=143 2024-11-12T04:09:31,177 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=144, ppid=143, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 285 msec 2024-11-12T04:09:31,177 INFO [PEWorker-2 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,tableCFs={testRemovePeerTableCFst4=null},bandwidth=0,serial=false 2024-11-12T04:09:31,178 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1480): Finished pid=143, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 435 msec 2024-11-12T04:09:31,198 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x1907368f to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@3d78a7bd 2024-11-12T04:09:31,210 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:31,210 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:31,210 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@266e343e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:31,211 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:31,218 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:31,218 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:31,218 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f80077 connected 2024-11-12T04:09:31,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=143 2024-11-12T04:09:31,368 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:09:31,368 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:31,369 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3960): Client=jenkins//172.17.0.2 update replication peer config, id=1, config=clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-12T04:09:31,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=145, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure 2024-11-12T04:09:31,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=145 2024-11-12T04:09:31,389 INFO [PEWorker-1 {}] hbase.HBaseConfiguration(308): Overriding client registry implementation to org.apache.hadoop.hbase.client.ZKConnectionRegistry 2024-11-12T04:09:31,390 INFO [PEWorker-1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:31,401 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:31,401 ERROR [PEWorker-1-EventThread {}] zookeeper.ClientCnxn$EventThread(581): Error while calling watcher. java.util.concurrent.RejectedExecutionException: Task org.apache.hadoop.hbase.trace.TraceUtil$$Lambda$361/0x00007f7288915890@4f3ec38c rejected from java.util.concurrent.ThreadPoolExecutor@a3c540f[Terminated, pool size = 0, active threads = 0, queued tasks = 0, completed tasks = 0] at java.util.concurrent.ThreadPoolExecutor$AbortPolicy.rejectedExecution(ThreadPoolExecutor.java:2065) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.reject(ThreadPoolExecutor.java:833) ~[?:?] at java.util.concurrent.ThreadPoolExecutor.execute(ThreadPoolExecutor.java:1365) ~[?:?] at java.util.concurrent.Executors$DelegatedExecutorService.execute(Executors.java:721) ~[?:?] at org.apache.hadoop.hbase.zookeeper.ZKWatcher.process(ZKWatcher.java:613) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.PendingWatcher.process(PendingWatcher.java:38) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.zookeeper.ClientCnxn$EventThread.processEvent(ClientCnxn.java:579) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:554) ~[zookeeper-3.8.4.jar:3.8.4] 2024-11-12T04:09:31,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=145 2024-11-12T04:09:31,509 DEBUG [PEWorker-1-EventThread {}] zookeeper.ZKWatcher(609): org.apache.hadoop.hbase.master.replication.ReplicationPeerManager@3f4290fdcheck-peer-cluster-id0x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:31,519 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:31,672 DEBUG [RSProcedureDispatcher-pool-2 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:31,674 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=146 2024-11-12T04:09:31,674 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=UPDATE_PEER_CONFIG 2024-11-12T04:09:31,677 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.ReplicationSourceManager(373): Terminate replication source for 1 2024-11-12T04:09:31,677 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Peer 1 state or config changed. Will close the previous replication source and open a new one 2024-11-12T04:09:31,678 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:31,678 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f80077, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:31,678 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f80077, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:31,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=145 2024-11-12T04:09:31,789 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f80077, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:31,790 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=146}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x1907368f to 127.0.0.1:50727 2024-11-12T04:09:31,790 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=146}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:31,811 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.ReplicationSource(229): queueId=1, ReplicationSource: 1, currentBandwidth=0 2024-11-12T04:09:31,811 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.ReplicationSourceManager(383): Startup replication source for 1 2024-11-12T04:09:31,812 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=146}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=146 2024-11-12T04:09:31,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=146 2024-11-12T04:09:31,813 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for UPDATE_CONFIG on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:31,815 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=146, resume processing ppid=145 2024-11-12T04:09:31,815 INFO [PEWorker-4 {}] replication.UpdatePeerConfigProcedure(197): Successfully updated peer config of 1 to clusterKey=127.0.0.1:50727:/hbase-test1,replicationEndpointImpl=null,replicateAllUserTables=false,bandwidth=0,serial=false 2024-11-12T04:09:31,815 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=146, ppid=145, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 294 msec 2024-11-12T04:09:31,816 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=145, state=SUCCESS; org.apache.hadoop.hbase.master.replication.UpdatePeerConfigProcedure in 446 msec 2024-11-12T04:09:31,839 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ReadOnlyZKClient(149): Connect 0x7a30741f to 127.0.0.1:50727 with session timeout=90000ms, retries=30, retry interval=1000ms, keepAlive=60000ms, zk client config=org.apache.zookeeper.client.ZKClientConfig@735ac471 2024-11-12T04:09:31,848 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(702): Retrieve cluster id failed java.util.concurrent.ExecutionException: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at java.util.concurrent.CompletableFuture.reportGet(CompletableFuture.java:396) ~[?:?] at java.util.concurrent.CompletableFuture.get(CompletableFuture.java:2073) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.retrieveClusterId(ConnectionImplementation.java:700) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionImplementation.(ConnectionImplementation.java:358) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$0(ConnectionFactory.java:425) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at java.security.AccessController.doPrivileged(AccessController.java:712) ~[?:?] at javax.security.auth.Subject.doAs(Subject.java:439) ~[?:?] at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1953) ~[hadoop-common-3.4.1.jar:?] at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:328) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.lambda$createConnection$2(ConnectionFactory.java:424) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:400) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:362) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.client.ConnectionFactory.createConnection(ConnectionFactory.java:156) ~[hbase-client-2.7.0-SNAPSHOT.jar:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.createConnection(HBaseInterClusterReplicationEndpoint.java:139) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.HBaseInterClusterReplicationEndpoint.init(HBaseInterClusterReplicationEndpoint.java:170) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initAndStartReplicationEndpoint(ReplicationSource.java:320) ~[classes/:?] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:539) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.zookeeper.KeeperException$NoNodeException: KeeperErrorCode = NoNode for /hbase-test1/hbaseid at org.apache.zookeeper.KeeperException.create(KeeperException.java:118) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.KeeperException.create(KeeperException.java:54) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$ZKTask$1.exec(ReadOnlyZKClient.java:185) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:386) ~[hbase-client-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] ... 1 more 2024-11-12T04:09:31,848 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] client.ConnectionImplementation(706): clusterid came back null, using default default-cluster 2024-11-12T04:09:31,848 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodecWithTags@5fe3ae31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T04:09:31,849 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.RecoverableZooKeeper(138): Process identifier=connection to cluster: 1 connecting to ZooKeeper ensemble=127.0.0.1:50727 2024-11-12T04:09:31,859 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 10x0, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T04:09:31,860 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): connection to cluster: 1-0x1012e84e2f8007a connected 2024-11-12T04:09:31,860 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(569): peerId=1, Could not connect to Peer ZK. Sleeping for 1000 millis 2024-11-12T04:09:31,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=145 2024-11-12T04:09:31,999 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: UPDATE_REPLICATION_PEER_CONFIG, peerId: 1 completed 2024-11-12T04:09:32,000 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3948): Client=jenkins//172.17.0.2 get replication peer config, id=1 2024-11-12T04:09:32,001 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-12T04:09:32,002 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(3926): Client=jenkins//172.17.0.2 removing replication peer, id=1 2024-11-12T04:09:32,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=147, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:32,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] procedure2.ProcedureExecutor(1098): Stored pid=148, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:32,003 DEBUG [PEWorker-1 {}] procedure.MasterProcedureScheduler(583): PEER '1', shared lock count=0, exclusively locked by procId=147 2024-11-12T04:09:32,004 DEBUG [PEWorker-1 {}] procedure2.ProcedureExecutor(1470): LOCK_EVENT_WAIT pid=148, state=RUNNABLE:PRE_PEER_MODIFICATION; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure 2024-11-12T04:09:32,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-11-12T04:09:32,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:09:32,007 DEBUG [PEWorker-2 {}] zookeeper.ZKUtil(111): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/peers/1/peer-state 2024-11-12T04:09:32,017 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1/peer-state 2024-11-12T04:09:32,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-12T04:09:32,018 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:36173-0x1012e84e2f80000, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/peers/1 2024-11-12T04:09:32,018 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1819): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure}] 2024-11-12T04:09:32,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-11-12T04:09:32,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:09:32,170 DEBUG [RSProcedureDispatcher-pool-0 {}] master.ServerManager(801): New admin connection to 354fa2a6db59,40009,1731384526494 2024-11-12T04:09:32,170 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009 {}] regionserver.RSRpcServices(3992): Executing remote procedure class org.apache.hadoop.hbase.replication.regionserver.RefreshPeerCallable, pid=149 2024-11-12T04:09:32,171 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.RefreshPeerCallable(45): Received a peer change event, peerId=1, type=REMOVE_PEER 2024-11-12T04:09:32,171 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.ReplicationSourceManager(287): Number of deleted recovered sources for 1: 0 2024-11-12T04:09:32,171 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.ReplicationSource(657): peerId=1, Closing source 1 because: Replication stream was removed by a user 2024-11-12T04:09:32,171 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] regionserver.ReplicationSource(508): peerId=1, Interrupted while sleeping between retries 2024-11-12T04:09:32,171 WARN [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKUtil(165): connection to cluster: 1-0x1012e84e2f8007a, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Unable to set watcher on znode (/hbase-test1/hbaseid) java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:32,171 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1 {}] zookeeper.ZKWatcher(734): connection to cluster: 1-0x1012e84e2f8007a, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received InterruptedException, will interrupt current thread and rethrow a SystemErrorException java.lang.InterruptedException: null at java.lang.Object.wait(Native Method) ~[?:?] at java.lang.Object.wait(Object.java:338) ~[?:?] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1604) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ClientCnxn.submitRequest(ClientCnxn.java:1576) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.zookeeper.ZooKeeper.exists(ZooKeeper.java:1864) ~[zookeeper-3.8.4.jar:3.8.4] at org.apache.hadoop.hbase.zookeeper.RecoverableZooKeeper.exists(RecoverableZooKeeper.java:250) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKUtil.checkExists(ZKUtil.java:158) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.readClusterIdZNode(ZKClusterId.java:59) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.zookeeper.ZKClusterId.getUUIDForCluster(ZKClusterId.java:89) ~[hbase-zookeeper-2.7.0-SNAPSHOT.jar:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.HBaseReplicationEndpoint.getPeerUUID(HBaseReplicationEndpoint.java:113) ~[classes/:2.7.0-SNAPSHOT] at org.apache.hadoop.hbase.replication.regionserver.ReplicationSource.initialize(ReplicationSource.java:566) ~[classes/:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:09:32,242 INFO [master/354fa2a6db59:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-12T04:09:32,242 INFO [master/354fa2a6db59:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-12T04:09:32,281 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0.replicationSource,1-EventThread {}] zookeeper.ZKWatcher(609): connection to cluster: 1-0x1012e84e2f8007a, quorum=127.0.0.1:50727, baseZNode=/hbase-test1 Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T04:09:32,281 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=149}] zookeeper.ReadOnlyZKClient(407): Close zookeeper connection 0x7a30741f to 127.0.0.1:50727 2024-11-12T04:09:32,281 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=149}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T04:09:32,281 INFO [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.ReplicationSourceManager(430): Done with the queue 1 2024-11-12T04:09:32,290 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=149}] zookeeper.ZKUtil(111): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Set watcher on existing znode=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1/354fa2a6db59%2C40009%2C1731384526494.1731384529497 2024-11-12T04:09:32,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1/354fa2a6db59%2C40009%2C1731384526494.1731384529497 2024-11-12T04:09:32,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1 2024-11-12T04:09:32,301 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:40009-0x1012e84e2f80001, quorum=127.0.0.1:50727, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/replication/rs/354fa2a6db59,40009,1731384526494/1 2024-11-12T04:09:32,301 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=149}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-12T04:09:32,301 DEBUG [RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0 {event_type=RS_REFRESH_PEER, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-12T04:09:32,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.HMaster(4106): Remote procedure done, pid=149 2024-11-12T04:09:32,303 INFO [PEWorker-3 {}] replication.RefreshPeerProcedure(119): Refresh peer 1 for REMOVE on 354fa2a6db59,40009,1731384526494 suceeded 2024-11-12T04:09:32,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1976): Finished subprocedure pid=149, resume processing ppid=147 2024-11-12T04:09:32,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1480): Finished pid=149, ppid=147, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RefreshPeerProcedure in 285 msec 2024-11-12T04:09:32,315 DEBUG [PEWorker-4 {}] replication.ZKReplicationQueueStorage(578): Peer /hbase/replication/hfile-refs/1 not found in hfile reference queue. 2024-11-12T04:09:32,315 INFO [PEWorker-4 {}] replication.RemovePeerProcedure(76): Successfully removed peer 1 2024-11-12T04:09:32,316 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1480): Finished pid=147, state=SUCCESS; org.apache.hadoop.hbase.master.replication.RemovePeerProcedure in 313 msec 2024-11-12T04:09:32,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=147 2024-11-12T04:09:32,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:09:32,318 INFO [RPCClient-NioEventLoopGroup-4-2 {}] client.RawAsyncHBaseAdmin$ReplicationProcedureBiConsumer(2971): Operation: REMOVE_REPLICATION_PEER, peerId: 1 completed 2024-11-12T04:09:32,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:09:33,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:09:34,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:09:35,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:09:36,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:09:37,799 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T04:09:40,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:09:44,867 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:09:45,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:09:50,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:09:50,715 DEBUG [master/354fa2a6db59:0.Chore.1 {}] balancer.RegionLocationFinder(172): Locality for region ca1f0981bae46f8dd99b7b3bc2a86bb7 changed from -1.0 to 0.0, refreshing cache 2024-11-12T04:10:00,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:10:05,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:10:10,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:10:14,868 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:10:20,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:10:25,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:10:40,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:10:44,868 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:10:45,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:11:00,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:11:05,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:11:14,868 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:11:20,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:11:25,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:11:40,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:11:44,869 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:11:45,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:12:00,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:12:05,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:12:14,869 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:12:20,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:12:26,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:12:40,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:12:44,869 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:12:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:13:01,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:13:06,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:13:14,870 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:13:21,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:13:26,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:13:41,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:13:44,870 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:13:46,506 INFO [LruBlockCacheStatsExecutor {}] hfile.LruBlockCache(1020): totalSize=880 MB, usedSize=660.99 KB, freeSize=879.35 MB, max=880 MB, blockCount=0, accesses=0, hits=0, hitRatio=0, cachingAccesses=0, cachingHits=0, cachingHitsRatio=0,evictions=29, evicted=0, evictedPerRun=0.0 2024-11-12T04:13:46,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:13:46,601 DEBUG [master/354fa2a6db59:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=0, created chunk count=7, reused chunk count=0, reuseRatio=0.00% 2024-11-12T04:13:46,602 DEBUG [master/354fa2a6db59:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0 2024-11-12T04:13:49,378 INFO [354fa2a6db59:40009Replication Statistics #0 {}] regionserver.Replication$ReplicationStatisticsTask(247): Global stats: WAL Edits Buffer Used=0B, Limit=268435456B 2024-11-12T04:13:51,394 DEBUG [master/354fa2a6db59:0.Chore.1 {}] balancer.BaseLoadBalancer(778): Start Generate Balance plan for cluster. 2024-11-12T04:13:51,395 DEBUG [master/354fa2a6db59:0.Chore.1 {}] master.HMaster(2144): Balancer is going into sleep until next period in 300000ms 2024-11-12T04:13:51,397 DEBUG [hconnection-0x989010-shared-pool-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T04:13:51,399 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:44586, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T04:13:51,408 DEBUG [master/354fa2a6db59:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-11-12T04:13:52,337 INFO [regionserver/354fa2a6db59:0.Chore.1 {}] regionserver.HRegionServer$PeriodicMemStoreFlusher(2070): MemstoreFlusherChore requesting flush of hbase:meta,,1.1588230740 because 1588230740/info has an old edit so flush to free WALs after random delay 283097 ms 2024-11-12T04:13:52,337 INFO [regionserver/354fa2a6db59:0.Chore.1 {}] regionserver.HRegionServer$PeriodicMemStoreFlusher(2070): MemstoreFlusherChore requesting flush of hbase:namespace,,1731384530724.ca1f0981bae46f8dd99b7b3bc2a86bb7. because ca1f0981bae46f8dd99b7b3bc2a86bb7/info has an old edit so flush to free WALs after random delay 31315 ms 2024-11-12T04:13:56,038 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T04:14:01,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:14:06,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:14:14,870 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:14:21,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:14:23,652 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing ca1f0981bae46f8dd99b7b3bc2a86bb7 1/1 column families, dataSize=78 B heapSize=488 B 2024-11-12T04:14:23,744 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/data/hbase/namespace/ca1f0981bae46f8dd99b7b3bc2a86bb7/.tmp/info/18f64fee50504b16bad74d71b655f82e is 45, key is default/info:d/1731384532085/Put/seqid=0 2024-11-12T04:14:23,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741837_1013 (size=5037) 2024-11-12T04:14:24,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=78 B at sequenceid=6 (bloomFilter=true), to=hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/data/hbase/namespace/ca1f0981bae46f8dd99b7b3bc2a86bb7/.tmp/info/18f64fee50504b16bad74d71b655f82e 2024-11-12T04:14:24,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/data/hbase/namespace/ca1f0981bae46f8dd99b7b3bc2a86bb7/.tmp/info/18f64fee50504b16bad74d71b655f82e as hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/data/hbase/namespace/ca1f0981bae46f8dd99b7b3bc2a86bb7/info/18f64fee50504b16bad74d71b655f82e 2024-11-12T04:14:24,210 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/data/hbase/namespace/ca1f0981bae46f8dd99b7b3bc2a86bb7/info/18f64fee50504b16bad74d71b655f82e, entries=2, sequenceid=6, filesize=4.9 K 2024-11-12T04:14:24,212 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~78 B/78, heapSize ~472 B/472, currentSize=0 B/0 for ca1f0981bae46f8dd99b7b3bc2a86bb7 in 560ms, sequenceid=6, compaction requested=false 2024-11-12T04:14:24,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for ca1f0981bae46f8dd99b7b3bc2a86bb7: 2024-11-12T04:14:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:14:41,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:14:44,871 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:14:46,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:14:51,987 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region ca1f0981bae46f8dd99b7b3bc2a86bb7, had cached 0 bytes from a total of 5037 2024-11-12T04:14:53,691 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:44407 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:44407,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:15:01,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:15:06,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:15:14,871 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:15:21,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:15:27,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:15:36,988 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region ca1f0981bae46f8dd99b7b3bc2a86bb7, had cached 0 bytes from a total of 5037 2024-11-12T04:15:42,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:15:44,871 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:15:47,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:16:02,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:16:07,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:16:14,872 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:16:21,988 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region ca1f0981bae46f8dd99b7b3bc2a86bb7, had cached 0 bytes from a total of 5037 2024-11-12T04:16:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:16:27,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:16:42,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:16:44,872 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:16:47,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:17:02,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:17:06,989 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region ca1f0981bae46f8dd99b7b3bc2a86bb7, had cached 0 bytes from a total of 5037 2024-11-12T04:17:07,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:17:14,872 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:17:22,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:17:27,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:17:42,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:17:44,873 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:17:47,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:17:51,989 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region ca1f0981bae46f8dd99b7b3bc2a86bb7, had cached 0 bytes from a total of 5037 2024-11-12T04:18:02,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:18:07,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:18:14,873 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:18:22,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:18:27,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:18:35,434 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-12T04:18:35,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2837): Flushing 1588230740 3/3 column families, dataSize=1.23 KB heapSize=2.87 KB 2024-11-12T04:18:35,456 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/data/hbase/meta/1588230740/.tmp/info/bb81f80b424848f9b458c8f097c404c0 is 143, key is hbase:namespace,,1731384530724.ca1f0981bae46f8dd99b7b3bc2a86bb7./info:regioninfo/1731384532012/Put/seqid=0 2024-11-12T04:18:35,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741838_1014 (size=6595) 2024-11-12T04:18:35,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.14 KB at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/data/hbase/meta/1588230740/.tmp/info/bb81f80b424848f9b458c8f097c404c0 2024-11-12T04:18:35,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/data/hbase/meta/1588230740/.tmp/table/7442f7659c584559a7b060fec426cc76 is 51, key is hbase:namespace/table:state/1731384532025/Put/seqid=0 2024-11-12T04:18:35,907 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741839_1015 (size=5242) 2024-11-12T04:18:35,907 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=94 B at sequenceid=9 (bloomFilter=true), to=hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/data/hbase/meta/1588230740/.tmp/table/7442f7659c584559a7b060fec426cc76 2024-11-12T04:18:35,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/data/hbase/meta/1588230740/.tmp/info/bb81f80b424848f9b458c8f097c404c0 as hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/data/hbase/meta/1588230740/info/bb81f80b424848f9b458c8f097c404c0 2024-11-12T04:18:35,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/data/hbase/meta/1588230740/info/bb81f80b424848f9b458c8f097c404c0, entries=10, sequenceid=9, filesize=6.4 K 2024-11-12T04:18:35,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/data/hbase/meta/1588230740/.tmp/table/7442f7659c584559a7b060fec426cc76 as hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/data/hbase/meta/1588230740/table/7442f7659c584559a7b060fec426cc76 2024-11-12T04:18:35,945 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/data/hbase/meta/1588230740/table/7442f7659c584559a7b060fec426cc76, entries=2, sequenceid=9, filesize=5.1 K 2024-11-12T04:18:35,947 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3040): Finished flush of dataSize ~1.23 KB/1264, heapSize ~2.59 KB/2648, currentSize=0 B/0 for 1588230740 in 512ms, sequenceid=9, compaction requested=false 2024-11-12T04:18:35,947 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2538): Flush status journal for 1588230740: 2024-11-12T04:18:36,990 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region ca1f0981bae46f8dd99b7b3bc2a86bb7, had cached 0 bytes from a total of 5037 2024-11-12T04:18:43,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:18:44,873 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:18:46,506 INFO [LruBlockCacheStatsExecutor {}] hfile.LruBlockCache(1020): totalSize=880 MB, usedSize=660.99 KB, freeSize=879.35 MB, max=880 MB, blockCount=0, accesses=0, hits=0, hitRatio=0, cachingAccesses=0, cachingHits=0, cachingHitsRatio=0,evictions=59, evicted=0, evictedPerRun=0.0 2024-11-12T04:18:46,601 DEBUG [master/354fa2a6db59:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=3, created chunk count=7, reused chunk count=0, reuseRatio=0.00% 2024-11-12T04:18:46,602 DEBUG [master/354fa2a6db59:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0 2024-11-12T04:18:48,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:18:49,196 DEBUG [master/354fa2a6db59:0.Chore.1 {}] replication.ZKReplicationQueueStorage(530): Didn't find a RegionServer that replicates, won't prevent deletions. 2024-11-12T04:18:49,378 INFO [354fa2a6db59:40009Replication Statistics #0 {}] regionserver.Replication$ReplicationStatisticsTask(247): Global stats: WAL Edits Buffer Used=0B, Limit=268435456B 2024-11-12T04:18:50,716 DEBUG [hconnection-0x989010-shared-pool-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T04:18:50,717 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:58396, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T04:18:50,733 DEBUG [master/354fa2a6db59:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-11-12T04:18:50,739 DEBUG [master/354fa2a6db59:0.Chore.1 {}] balancer.BaseLoadBalancer(778): Start Generate Balance plan for cluster. 2024-11-12T04:18:50,740 DEBUG [master/354fa2a6db59:0.Chore.1 {}] master.HMaster(2144): Balancer is going into sleep until next period in 300000ms 2024-11-12T04:18:56,040 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T04:19:03,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:19:05,444 ERROR [LeaseRenewer:jenkins.hfs.0@localhost:44407 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins.hfs.0@localhost:44407,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:19:08,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:19:14,874 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:19:20,517 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837 2024-11-12T04:19:21,990 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region ca1f0981bae46f8dd99b7b3bc2a86bb7, had cached 0 bytes from a total of 5037 2024-11-12T04:19:23,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:19:28,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:19:43,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:19:44,874 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:19:48,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:20:03,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:20:05,518 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837 2024-11-12T04:20:06,991 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region ca1f0981bae46f8dd99b7b3bc2a86bb7, had cached 0 bytes from a total of 5037 2024-11-12T04:20:08,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:20:14,874 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:20:23,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:20:28,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:20:43,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:20:44,875 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:20:48,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:20:50,518 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837 2024-11-12T04:20:51,991 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region ca1f0981bae46f8dd99b7b3bc2a86bb7, had cached 0 bytes from a total of 5037 2024-11-12T04:21:03,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:21:09,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:21:14,875 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:21:24,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:21:29,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:21:35,519 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837 2024-11-12T04:21:36,992 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region ca1f0981bae46f8dd99b7b3bc2a86bb7, had cached 0 bytes from a total of 5037 2024-11-12T04:21:44,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:21:44,875 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:21:49,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:22:04,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:22:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:22:14,875 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:22:20,520 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837 2024-11-12T04:22:21,992 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region ca1f0981bae46f8dd99b7b3bc2a86bb7, had cached 0 bytes from a total of 5037 2024-11-12T04:22:24,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:22:29,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:22:44,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:22:44,875 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:22:49,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:23:04,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:23:05,520 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837 2024-11-12T04:23:06,993 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region ca1f0981bae46f8dd99b7b3bc2a86bb7, had cached 0 bytes from a total of 5037 2024-11-12T04:23:09,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:23:14,876 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:23:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:23:29,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:23:44,876 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:23:44,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:23:46,506 INFO [LruBlockCacheStatsExecutor {}] hfile.LruBlockCache(1020): totalSize=880 MB, usedSize=663.48 KB, freeSize=879.35 MB, max=880 MB, blockCount=2, accesses=2, hits=0, hitRatio=0, cachingAccesses=2, cachingHits=0, cachingHitsRatio=0,evictions=89, evicted=0, evictedPerRun=0.0 2024-11-12T04:23:46,601 DEBUG [master/354fa2a6db59:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=3, created chunk count=7, reused chunk count=0, reuseRatio=0.00% 2024-11-12T04:23:46,602 DEBUG [master/354fa2a6db59:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0 2024-11-12T04:23:48,670 DEBUG [master:store-WAL-Roller {}] wal.AbstractWALRoller(200): WAL AsyncFSWAL 354fa2a6db59%2C36173%2C1731384525744:(num 1731384528511) roll period 900000 ms elapsed 2024-11-12T04:23:48,687 DEBUG [master:store-WAL-Roller {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(599): When create output stream for /user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/WALs/354fa2a6db59,36173,1731384525744/354fa2a6db59%2C36173%2C1731384525744.1731385428670, exclude list is [], retry=0 2024-11-12T04:23:48,693 DEBUG [RS-EventLoopGroup-3-3 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:34483,DS-ec3ff3ff-91c2-401f-ac91-ee43951db131,DISK] 2024-11-12T04:23:48,696 INFO [master:store-Flusher {}] regionserver.HRegion(2837): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=428.68 KB heapSize=530.64 KB 2024-11-12T04:23:48,696 INFO [master:store-WAL-Roller {}] wal.AbstractFSWAL(837): Rolled WAL /user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/WALs/354fa2a6db59,36173,1731384525744/354fa2a6db59%2C36173%2C1731384525744.1731384528511 with entries=1160, filesize=503.08 KB; new WAL /user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/WALs/354fa2a6db59,36173,1731384525744/354fa2a6db59%2C36173%2C1731384525744.1731385428670 2024-11-12T04:23:48,697 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(925): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:41335:41335)] 2024-11-12T04:23:48,697 DEBUG [master:store-WAL-Roller {}] wal.AbstractFSWAL(751): hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/WALs/354fa2a6db59,36173,1731384525744/354fa2a6db59%2C36173%2C1731384525744.1731384528511 is not closed yet, will try archiving it next time 2024-11-12T04:23:48,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741830_1006 (size=515161) 2024-11-12T04:23:48,719 DEBUG [master:store-Flusher {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c58d6b39d3114313b2527527efe258cc is 82, key is hbase:meta,,1/info:regioninfo/1731384530564/Put/seqid=0 2024-11-12T04:23:48,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741841_1017 (size=5672) 2024-11-12T04:23:49,126 INFO [master:store-Flusher {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=1163 (bloomFilter=true), to=hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c58d6b39d3114313b2527527efe258cc 2024-11-12T04:23:49,159 DEBUG [master:store-Flusher {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b92b1c7729f447d7b4d882c883943d86 is 293, key is \x00\x00\x00\x00\x00\x00\x00\x94/proc:d/1731384572002/Put/seqid=0 2024-11-12T04:23:49,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741842_1018 (size=11538) 2024-11-12T04:23:49,165 INFO [master:store-Flusher {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=427.13 KB at sequenceid=1163 (bloomFilter=true), to=hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b92b1c7729f447d7b4d882c883943d86 2024-11-12T04:23:49,173 INFO [master:store-Flusher {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b92b1c7729f447d7b4d882c883943d86 2024-11-12T04:23:49,195 DEBUG [master:store-Flusher {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/780cfcd4d9524481b830491e5b6cded8 is 69, key is 354fa2a6db59,40009,1731384526494/rs:state/1731384529271/Put/seqid=0 2024-11-12T04:23:49,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741843_1019 (size=5156) 2024-11-12T04:23:49,378 INFO [354fa2a6db59:40009Replication Statistics #0 {}] regionserver.Replication$ReplicationStatisticsTask(247): Global stats: WAL Edits Buffer Used=0B, Limit=268435456B 2024-11-12T04:23:49,610 INFO [master:store-Flusher {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=1163 (bloomFilter=true), to=hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/780cfcd4d9524481b830491e5b6cded8 2024-11-12T04:23:49,633 DEBUG [master:store-Flusher {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b3c4fa7278974bbaac4f0d1ceee92e74 is 68, key is replication_peer_modification_on/state:d/1731384567540/Put/seqid=0 2024-11-12T04:23:49,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:34483 is added to blk_1073741844_1020 (size=5154) 2024-11-12T04:23:50,038 INFO [master:store-Flusher {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1 KB at sequenceid=1163 (bloomFilter=true), to=hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b3c4fa7278974bbaac4f0d1ceee92e74 2024-11-12T04:23:50,046 DEBUG [master:store-Flusher {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/c58d6b39d3114313b2527527efe258cc as hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c58d6b39d3114313b2527527efe258cc 2024-11-12T04:23:50,053 INFO [master:store-Flusher {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/c58d6b39d3114313b2527527efe258cc, entries=8, sequenceid=1163, filesize=5.5 K 2024-11-12T04:23:50,055 DEBUG [master:store-Flusher {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/b92b1c7729f447d7b4d882c883943d86 as hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b92b1c7729f447d7b4d882c883943d86 2024-11-12T04:23:50,062 INFO [master:store-Flusher {}] regionserver.StoreFileReader(539): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for b92b1c7729f447d7b4d882c883943d86 2024-11-12T04:23:50,062 INFO [master:store-Flusher {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/b92b1c7729f447d7b4d882c883943d86, entries=149, sequenceid=1163, filesize=11.3 K 2024-11-12T04:23:50,063 DEBUG [master:store-Flusher {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/780cfcd4d9524481b830491e5b6cded8 as hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/780cfcd4d9524481b830491e5b6cded8 2024-11-12T04:23:50,070 INFO [master:store-Flusher {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/780cfcd4d9524481b830491e5b6cded8, entries=1, sequenceid=1163, filesize=5.0 K 2024-11-12T04:23:50,072 DEBUG [master:store-Flusher {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/state/b3c4fa7278974bbaac4f0d1ceee92e74 as hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/b3c4fa7278974bbaac4f0d1ceee92e74 2024-11-12T04:23:50,079 INFO [master:store-Flusher {}] regionserver.HStore$StoreFlusherImpl(1989): Added hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/state/b3c4fa7278974bbaac4f0d1ceee92e74, entries=1, sequenceid=1163, filesize=5.0 K 2024-11-12T04:23:50,080 INFO [master:store-Flusher {}] regionserver.HRegion(3040): Finished flush of dataSize ~428.68 KB/438973, heapSize ~530.58 KB/543312, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1383ms, sequenceid=1163, compaction requested=false 2024-11-12T04:23:50,080 DEBUG [master:store-Flusher {}] regionserver.HRegion(2538): Flush status journal for 1595e783b53d99cd5eef43b6debb2682: 2024-11-12T04:23:50,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:23:50,521 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837 2024-11-12T04:23:51,993 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region ca1f0981bae46f8dd99b7b3bc2a86bb7, had cached 0 bytes from a total of 5037 2024-11-12T04:23:53,685 DEBUG [master/354fa2a6db59:0.Chore.1 {}] balancer.BaseLoadBalancer(778): Start Generate Balance plan for cluster. 2024-11-12T04:23:53,685 DEBUG [master/354fa2a6db59:0.Chore.1 {}] master.HMaster(2144): Balancer is going into sleep until next period in 300000ms 2024-11-12T04:23:53,686 DEBUG [hconnection-0x989010-shared-pool-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T04:23:53,687 INFO [RS-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47190, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T04:23:53,692 DEBUG [master/354fa2a6db59:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-11-12T04:23:56,028 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T04:24:05,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:24:10,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:24:14,876 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:24:18,693 ERROR [LeaseRenewer:jenkins@localhost:44407 {}] server.NIOServerCnxnFactory(85): Thread Thread[LeaseRenewer:jenkins@localhost:44407,5,FailOnTimeoutGroup] died java.lang.NullPointerException: Cannot invoke "org.apache.hadoop.hdfs.DFSOutputStream.getNamespace()" because "outputStream" is null at org.apache.hadoop.hdfs.DFSClient.getNamespaces(DFSClient.java:596) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.DFSClient.renewLease(DFSClient.java:618) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.renew(LeaseRenewer.java:425) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.run(LeaseRenewer.java:445) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer.access$800(LeaseRenewer.java:77) ~[hadoop-hdfs-client-3.4.1.jar:?] at org.apache.hadoop.hdfs.client.impl.LeaseRenewer$1.run(LeaseRenewer.java:336) ~[hadoop-hdfs-client-3.4.1.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T04:24:25,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:24:30,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:24:35,521 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837 2024-11-12T04:24:36,993 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region ca1f0981bae46f8dd99b7b3bc2a86bb7, had cached 0 bytes from a total of 5037 2024-11-12T04:24:44,877 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:24:45,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:24:50,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:25:05,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:25:10,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:25:14,878 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:25:20,521 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837 2024-11-12T04:25:21,994 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region ca1f0981bae46f8dd99b7b3bc2a86bb7, had cached 0 bytes from a total of 5037 2024-11-12T04:25:25,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:25:30,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:25:44,879 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:25:45,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:25:50,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:26:05,522 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837 2024-11-12T04:26:05,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:26:06,994 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region ca1f0981bae46f8dd99b7b3bc2a86bb7, had cached 0 bytes from a total of 5037 2024-11-12T04:26:10,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:26:14,879 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:26:25,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:26:30,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:26:44,880 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:26:45,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:26:50,522 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837 2024-11-12T04:26:50,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:26:51,995 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region ca1f0981bae46f8dd99b7b3bc2a86bb7, had cached 0 bytes from a total of 5037 2024-11-12T04:27:05,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:27:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:27:14,880 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:27:25,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:27:31,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:27:35,523 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837 2024-11-12T04:27:36,995 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region ca1f0981bae46f8dd99b7b3bc2a86bb7, had cached 0 bytes from a total of 5037 2024-11-12T04:27:44,880 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:27:46,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:27:51,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:28:06,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:28:11,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:28:14,881 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:28:20,523 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837 2024-11-12T04:28:21,996 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region ca1f0981bae46f8dd99b7b3bc2a86bb7, had cached 0 bytes from a total of 5037 2024-11-12T04:28:26,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:28:31,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:28:44,881 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:28:46,507 INFO [LruBlockCacheStatsExecutor {}] hfile.LruBlockCache(1020): totalSize=880 MB, usedSize=663.48 KB, freeSize=879.35 MB, max=880 MB, blockCount=2, accesses=4, hits=2, hitRatio=50.00%, , cachingAccesses=4, cachingHits=2, cachingHitsRatio=50.00%, evictions=119, evicted=0, evictedPerRun=0.0 2024-11-12T04:28:46,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:28:46,601 DEBUG [master/354fa2a6db59:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=6, created chunk count=7, reused chunk count=1, reuseRatio=12.50% 2024-11-12T04:28:46,602 DEBUG [master/354fa2a6db59:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0 2024-11-12T04:28:49,378 INFO [354fa2a6db59:40009Replication Statistics #0 {}] regionserver.Replication$ReplicationStatisticsTask(247): Global stats: WAL Edits Buffer Used=0B, Limit=268435456B 2024-11-12T04:28:51,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:28:52,683 DEBUG [master/354fa2a6db59:0.Chore.1 {}] balancer.BaseLoadBalancer(778): Start Generate Balance plan for cluster. 2024-11-12T04:28:52,684 DEBUG [master/354fa2a6db59:0.Chore.1 {}] master.HMaster(2144): Balancer is going into sleep until next period in 300000ms 2024-11-12T04:28:52,685 DEBUG [master/354fa2a6db59:0.Chore.1 {}] replication.ZKReplicationQueueStorage(530): Didn't find a RegionServer that replicates, won't prevent deletions. 2024-11-12T04:28:52,688 DEBUG [hconnection-0x989010-shared-pool-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T04:28:52,689 INFO [RS-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41062, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T04:28:52,692 DEBUG [master/354fa2a6db59:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-11-12T04:28:56,040 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T04:29:05,524 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837 2024-11-12T04:29:06,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:29:06,996 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region ca1f0981bae46f8dd99b7b3bc2a86bb7, had cached 0 bytes from a total of 5037 2024-11-12T04:29:11,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:29:14,881 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:29:26,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:29:31,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:29:44,882 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:29:46,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:29:50,524 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837 2024-11-12T04:29:51,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:29:51,997 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region ca1f0981bae46f8dd99b7b3bc2a86bb7, had cached 0 bytes from a total of 5037 2024-11-12T04:30:06,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:30:11,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:30:14,882 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:30:27,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:30:32,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:30:35,525 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837 2024-11-12T04:30:36,997 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region ca1f0981bae46f8dd99b7b3bc2a86bb7, had cached 0 bytes from a total of 5037 2024-11-12T04:30:44,882 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:30:47,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:30:52,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:31:07,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:31:12,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:31:14,883 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:31:20,525 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837 2024-11-12T04:31:21,998 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region ca1f0981bae46f8dd99b7b3bc2a86bb7, had cached 0 bytes from a total of 5037 2024-11-12T04:31:27,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:31:32,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:31:44,883 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:31:47,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:31:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:32:05,526 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837 2024-11-12T04:32:06,998 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region ca1f0981bae46f8dd99b7b3bc2a86bb7, had cached 0 bytes from a total of 5037 2024-11-12T04:32:07,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:32:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:32:14,884 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:32:27,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:32:32,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:32:44,884 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:32:47,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:32:50,527 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837 2024-11-12T04:32:51,999 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region ca1f0981bae46f8dd99b7b3bc2a86bb7, had cached 0 bytes from a total of 5037 2024-11-12T04:32:52,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:33:07,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:33:12,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:33:14,884 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:33:28,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:33:33,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:33:35,527 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837 2024-11-12T04:33:36,999 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region ca1f0981bae46f8dd99b7b3bc2a86bb7, had cached 0 bytes from a total of 5037 2024-11-12T04:33:44,885 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:33:46,507 INFO [LruBlockCacheStatsExecutor {}] hfile.LruBlockCache(1020): totalSize=880 MB, usedSize=663.48 KB, freeSize=879.35 MB, max=880 MB, blockCount=2, accesses=6, hits=4, hitRatio=66.67%, , cachingAccesses=6, cachingHits=4, cachingHitsRatio=66.67%, evictions=149, evicted=0, evictedPerRun=0.0 2024-11-12T04:33:46,602 DEBUG [master/354fa2a6db59:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): data stats (chunk size=2097152): current pool size=6, created chunk count=7, reused chunk count=1, reuseRatio=12.50% 2024-11-12T04:33:46,602 DEBUG [master/354fa2a6db59:0:becomeActiveMaster-MemStoreChunkPool Statistics {}] regionserver.ChunkCreator$MemStoreChunkPool$StatisticsThread(417): index stats (chunk size=209715): current pool size=0, created chunk count=0, reused chunk count=0, reuseRatio=0 2024-11-12T04:33:48,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:33:49,378 INFO [354fa2a6db59:40009Replication Statistics #0 {}] regionserver.Replication$ReplicationStatisticsTask(247): Global stats: WAL Edits Buffer Used=0B, Limit=268435456B 2024-11-12T04:33:52,333 DEBUG [master/354fa2a6db59:0.Chore.1 {}] balancer.BaseLoadBalancer(778): Start Generate Balance plan for cluster. 2024-11-12T04:33:52,333 DEBUG [master/354fa2a6db59:0.Chore.1 {}] master.HMaster(2144): Balancer is going into sleep until next period in 300000ms 2024-11-12T04:33:52,334 DEBUG [hconnection-0x989010-shared-pool-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T04:33:52,336 INFO [RS-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:60518, version=2.7.0-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T04:33:52,342 DEBUG [master/354fa2a6db59:0.Chore.1 {}] janitor.CatalogJanitor(179): 2024-11-12T04:33:53,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:33:56,040 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T04:34:08,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:34:13,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 2024-11-12T04:34:14,885 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer(620): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T04:34:20,528 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region 1588230740, had cached 0 bytes from a total of 11837 2024-11-12T04:34:22,000 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsRegionWrapperImpl$HRegionMetricsWrapperRunnable(324): Region ca1f0981bae46f8dd99b7b3bc2a86bb7, had cached 0 bytes from a total of 5037 2024-11-12T04:34:28,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=148 2024-11-12T04:34:33,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173 {}] master.MasterRpcServices(1305): Checking to see if procedure is done pid=23 ====> TEST TIMED OUT. PRINTING THREAD DUMP. <==== Timestamp: 2024-11-12 04:34:41,407 "pool-19-thread-1" prio=5 tid=103 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "BootstrapNodeManager" daemon prio=5 tid=257 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Signal Dispatcher" daemon prio=9 tid=4 runnable java.lang.Thread.State: RUNNABLE "qtp1312628392-40" daemon prio=5 tid=40 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x00007f7288423bb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-4" daemon prio=5 tid=159 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "OldWALsCleaner-0" daemon prio=5 tid=259 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.hadoop.hbase.master.cleaner.LogCleaner.deleteFile(LogCleaner.java:172) at app//org.apache.hadoop.hbase.master.cleaner.LogCleaner.lambda$createOldWalsCleaner$1(LogCleaner.java:152) at app//org.apache.hadoop.hbase.master.cleaner.LogCleaner$$Lambda$608/0x00007f7288aa1458.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "region-location-2" daemon prio=5 tid=744 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RS_OPEN_META-regionserver/354fa2a6db59:0-0" daemon prio=5 tid=294 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-9" daemon prio=5 tid=165 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "PEWorker-2" daemon prio=5 tid=246 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149) at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2128) "Hadoop-Metrics-Updater-0" daemon prio=5 tid=58 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server listener on 0" daemon prio=5 tid=54 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hadoop.ipc.Server$Listener.run(Server.java:1559) "Async-Client-Retry-Timer-pool-0" daemon prio=5 tid=319 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) at app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=36173" daemon prio=5 tid=178 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:71) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "qtp764466342-88-acceptor-0@716f7bf9-ServerConnector@2117487{HTTP/1.1, (http/1.1)}{localhost:33505}" daemon prio=3 tid=88 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.Net.accept(Native Method) at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425) at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:704) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RS:0;354fa2a6db59:40009" daemon prio=5 tid=206 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:81) at app//org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:64) at app//org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:1164) at app//org.apache.hadoop.hbase.MiniHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(MiniHBaseCluster.java:176) at app//org.apache.hadoop.hbase.MiniHBaseCluster$MiniHBaseClusterRegionServer.access$000(MiniHBaseCluster.java:124) at app//org.apache.hadoop.hbase.MiniHBaseCluster$MiniHBaseClusterRegionServer$1.run(MiniHBaseCluster.java:160) at java.base@17.0.11/java.security.AccessController.executePrivileged(AccessController.java:776) at java.base@17.0.11/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base@17.0.11/javax.security.auth.Subject.doAs(Subject.java:376) at app//org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at app//org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at app//org.apache.hadoop.hbase.MiniHBaseCluster$MiniHBaseClusterRegionServer.run(MiniHBaseCluster.java:157) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "AsyncFSWAL-0-hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300-prefix:354fa2a6db59,40009,1731384526494.meta" daemon prio=5 tid=297 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Time-limited test.LruBlockCache.EvictionThread" daemon prio=5 tid=187 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.hbase.io.hfile.LruBlockCache$EvictionThread.run(LruBlockCache.java:957) "NIOServerCxnFactory.SelectorThread-1" daemon prio=5 tid=139 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403) at app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368) "Monitor thread for TaskMonitor" daemon prio=5 tid=209 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hbase.monitoring.TaskMonitor$MonitorRunnable.run(TaskMonitor.java:325) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "qtp1312628392-37" daemon prio=5 tid=37 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x00007f7288423bb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Finalizer" daemon prio=8 tid=3 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) at java.base@17.0.11/java.lang.ref.Finalizer$FinalizerThread.run(Finalizer.java:172) "IPC Server idle connection scanner for port 44407" daemon prio=5 tid=56 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) at java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) "SessionTracker" daemon prio=5 tid=141 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.zookeeper.server.SessionTrackerImpl.run(SessionTrackerImpl.java:163) "WorkerMonitor" daemon prio=5 tid=244 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:279) at app//org.apache.hadoop.hbase.procedure2.util.DelayedUtil.takeWithoutInterrupt(DelayedUtil.java:81) at app//org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:56) "IPC Server handler 3 on default port 45791" daemon prio=5 tid=107 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "regionserver/354fa2a6db59:0.leaseChecker" daemon prio=5 tid=274 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hbase.regionserver.LeaseManager.run(LeaseManager.java:82) "master:store-Flusher" daemon prio=5 tid=240 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1759) at app//org.apache.hadoop.hbase.master.region.MasterRegionFlusherAndCompactor.flushLoop(MasterRegionFlusherAndCompactor.java:200) at app//org.apache.hadoop.hbase.master.region.MasterRegionFlusherAndCompactor$$Lambda$480/0x00007f72889da518.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Timer for 'HBase' metrics system" daemon prio=5 tid=835 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) at java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) "RS-EventLoopGroup-3-3" daemon prio=10 tid=293 runnable java.lang.Thread.State: RUNNABLE at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait0(Native Method) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:193) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:304) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:368) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "PEWorker-3" daemon prio=5 tid=247 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149) at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2128) "NIOWorkerThread-8" daemon prio=5 tid=164 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-15" daemon prio=5 tid=171 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Time-limited test-EventThread" daemon prio=5 tid=156 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) "NIOWorkerThread-14" daemon prio=5 tid=170 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "surefire-forkedjvm-stream-flusher" daemon prio=5 tid=16 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOServerCxnFactory.SelectorThread-0" daemon prio=5 tid=138 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.select(NIOServerCnxnFactory.java:403) at app//org.apache.zookeeper.server.NIOServerCnxnFactory$SelectorThread.run(NIOServerCnxnFactory.java:368) "qtp1312628392-39" daemon prio=5 tid=39 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x00007f7288423bb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "354fa2a6db59:36173" daemon prio=5 tid=251 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at app//org.apache.hadoop.hbase.master.assignment.AssignmentManager.waitOnAssignQueue(AssignmentManager.java:2370) at app//org.apache.hadoop.hbase.master.assignment.AssignmentManager.processAssignQueue(AssignmentManager.java:2392) at app//org.apache.hadoop.hbase.master.assignment.AssignmentManager.access$600(AssignmentManager.java:112) at app//org.apache.hadoop.hbase.master.assignment.AssignmentManager$1.run(AssignmentManager.java:2332) "surefire-forkedjvm-command-thread" daemon prio=5 tid=18 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:230) at java.base@17.0.11/sun.nio.ch.PendingFuture.get(PendingFuture.java:178) at app//org.apache.maven.surefire.api.util.internal.Channels$2.read(Channels.java:127) at java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) at java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) at java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) at app//org.apache.maven.surefire.api.util.internal.Channels$3.readImpl(Channels.java:169) at app//org.apache.maven.surefire.api.util.internal.AbstractNoninterruptibleReadableChannel.read(AbstractNoninterruptibleReadableChannel.java:50) at app//org.apache.maven.surefire.api.stream.AbstractStreamDecoder.read(AbstractStreamDecoder.java:430) at app//org.apache.maven.surefire.api.stream.AbstractStreamDecoder.read(AbstractStreamDecoder.java:419) at app//org.apache.maven.surefire.api.stream.AbstractStreamDecoder.readMessageType(AbstractStreamDecoder.java:116) at app//org.apache.maven.surefire.booter.stream.CommandDecoder.decode(CommandDecoder.java:77) at app//org.apache.maven.surefire.booter.spi.CommandChannelDecoder.decode(CommandChannelDecoder.java:60) at app//org.apache.maven.surefire.booter.CommandReader$CommandRunnable.run(CommandReader.java:290) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RS:0;354fa2a6db59:40009-longCompactions-0" daemon prio=5 tid=272 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at app//org.apache.hadoop.hbase.util.StealJobQueue.take(StealJobQueue.java:101) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "JvmPauseMonitor" daemon prio=5 tid=271 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hbase.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:148) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Time-limited test-SendThread(127.0.0.1:50727)" daemon prio=5 tid=155 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) "Session-HouseKeeper-67bf5572-1" prio=5 tid=45 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor@6132670e" daemon prio=5 tid=71 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.server.namenode.LeaseManager$Monitor.run(LeaseManager.java:537) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "qtp1312628392-41-acceptor-0@53176e76-ServerConnector@1f7e2804{HTTP/1.1, (http/1.1)}{localhost:45153}" daemon prio=3 tid=41 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.Net.accept(Native Method) at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425) at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391) at app//org.eclipse.jetty.server.ServerConnector.accept(ServerConnector.java:388) at app//org.eclipse.jetty.server.AbstractConnector$Acceptor.run(AbstractConnector.java:704) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RS-EventLoopGroup-1-1" daemon prio=10 tid=154 runnable java.lang.Thread.State: RUNNABLE at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "zk-event-processor-pool-0" daemon prio=5 tid=405 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40009" daemon prio=5 tid=198 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:71) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "PacketResponder: BP-1580922183-172.17.0.2-1731384522326:blk_1073741840_1016, type=LAST_IN_PIPELINE" daemon prio=5 tid=793 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.lang.Object.wait(Object.java:338) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "SnapshotHandlerChoreCleaner" daemon prio=5 tid=253 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Notification Thread" daemon prio=9 tid=13 runnable java.lang.Thread.State: RUNNABLE "RpcServer.metaPriority.FPBQ.Fifo.handler=0,queue=0,port=40009" daemon prio=5 tid=204 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "354fa2a6db59:40009Replication Statistics #0" daemon prio=5 tid=279 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Time-limited test-SendThread(127.0.0.1:50727)" daemon prio=5 tid=403 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) "NIOWorkerThread-1" daemon prio=5 tid=145 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/cluster_b0a2b79c-cac7-9f39-3098-f01052b75b38/dfs/data/data2/current/BP-1580922183-172.17.0.2-1731384522326" daemon prio=5 tid=124 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:225) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ProcExecTimeout" daemon prio=5 tid=243 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:279) at app//org.apache.hadoop.hbase.procedure2.util.DelayedUtil.takeWithoutInterrupt(DelayedUtil.java:81) at app//org.apache.hadoop.hbase.procedure2.TimeoutExecutorThread.run(TimeoutExecutorThread.java:56) "org.apache.hadoop.hdfs.server.datanode.DataXceiverServer@6a67db2f" daemon prio=5 tid=85 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.Net.accept(Native Method) at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.implAccept(ServerSocketChannelImpl.java:425) at java.base@17.0.11/sun.nio.ch.ServerSocketChannelImpl.accept(ServerSocketChannelImpl.java:391) at java.base@17.0.11/sun.nio.ch.ServerSocketAdaptor.accept(ServerSocketAdaptor.java:126) at app//org.apache.hadoop.hdfs.net.TcpPeerServer.accept(TcpPeerServer.java:85) at app//org.apache.hadoop.hdfs.server.datanode.DataXceiverServer.run(DataXceiverServer.java:242) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Time-limited test.named-queue-events-pool-0" daemon prio=5 tid=152 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at app//com.lmax.disruptor.BlockingWaitStrategy.waitFor(BlockingWaitStrategy.java:47) at app//com.lmax.disruptor.ProcessingSequenceBarrier.waitFor(ProcessingSequenceBarrier.java:56) at app//com.lmax.disruptor.BatchEventProcessor.processEvents(BatchEventProcessor.java:159) at app//com.lmax.disruptor.BatchEventProcessor.run(BatchEventProcessor.java:125) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.replication.FPBQ.Fifo.handler=1,queue=0,port=40009" daemon prio=5 tid=202 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "ProcessThread(sid:0 cport:50727):" daemon prio=5 tid=143 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.zookeeper.server.PrepRequestProcessor.run(PrepRequestProcessor.java:142) "org.apache.hadoop.hdfs.server.blockmanagement.PendingReconstructionBlocks$PendingReconstructionMonitor@5ec1d4f9" daemon prio=5 tid=61 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.server.blockmanagement.PendingReconstructionBlocks$PendingReconstructionMonitor.run(PendingReconstructionBlocks.java:267) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "region-location-0" daemon prio=5 tid=317 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.replication.FPBQ.Fifo.handler=2,queue=0,port=40009" daemon prio=5 tid=203 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "master/354fa2a6db59:0:becomeActiveMaster-MemStoreChunkPool Statistics" daemon prio=5 tid=211 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-6" daemon prio=5 tid=162 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "AsyncFSWAL-0-hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300/MasterData-prefix:354fa2a6db59,36173,1731384525744" daemon prio=5 tid=237 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server handler 1 on default port 45791" daemon prio=5 tid=105 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "SSL Certificates Store Monitor" daemon prio=5 tid=25 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.lang.Object.wait(Object.java:338) at java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:537) at java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) "RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=36173" daemon prio=5 tid=177 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:71) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "RPCClient-NioEventLoopGroup-4-1" daemon prio=5 tid=324 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RequestThrottler" daemon prio=5 tid=144 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.zookeeper.server.RequestThrottler.run(RequestThrottler.java:147) "DatanodeAdminMonitor-0" daemon prio=5 tid=62 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "LruBlockCacheStatsExecutor" daemon prio=5 tid=189 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "qtp1312628392-38" daemon prio=5 tid=38 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x00007f7288423bb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Time-limited test-SendThread(127.0.0.1:50727)" daemon prio=5 tid=192 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.zookeeper.ClientCnxnSocketNIO.doTransport(ClientCnxnSocketNIO.java:332) at app//org.apache.zookeeper.ClientCnxn$SendThread.run(ClientCnxn.java:1289) "master/354fa2a6db59:0:becomeActiveMaster-HFileCleaner.large.0-1731384529156" daemon prio=5 tid=266 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at app//org.apache.hadoop.hbase.util.StealJobQueue.take(StealJobQueue.java:101) at app//org.apache.hadoop.hbase.master.cleaner.HFileCleaner.consumerLoop(HFileCleaner.java:285) at app//org.apache.hadoop.hbase.master.cleaner.HFileCleaner$1.run(HFileCleaner.java:254) "IPC Server Responder" daemon prio=5 tid=57 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:1733) at app//org.apache.hadoop.ipc.Server$Responder.run(Server.java:1716) "java.util.concurrent.ThreadPoolExecutor$Worker@40cb6f02[State = -1, empty queue]" daemon prio=5 tid=135 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ConnnectionExpirer" daemon prio=5 tid=137 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.zookeeper.server.NIOServerCnxnFactory$ConnectionExpirerThread.run(NIOServerCnxnFactory.java:554) "org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor@6b49abe4" daemon prio=5 tid=72 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeResourceMonitor.run(FSNamesystem.java:4550) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Command processor" daemon prio=5 tid=101 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.processQueue(BPServiceActor.java:1411) at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor$CommandProcessingThread.run(BPServiceActor.java:1395) "DataXceiver for client DFSClient_NONMAPREDUCE_1244051054_22 at /127.0.0.1:39614 [Receiving block BP-1580922183-172.17.0.2-1731384522326:blk_1073741834_1010]" daemon prio=5 tid=295 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) at app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) at java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) at java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) at java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) at java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) at app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "region-location-1" daemon prio=5 tid=318 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "master/354fa2a6db59:0:becomeActiveMaster-HFileCleaner.small.0-1731384529163" daemon prio=5 tid=267 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) at app//org.apache.hadoop.hbase.master.cleaner.HFileCleaner.consumerLoop(HFileCleaner.java:285) at app//org.apache.hadoop.hbase.master.cleaner.HFileCleaner$2.run(HFileCleaner.java:269) "regionserver/354fa2a6db59:0.procedureResultReporter" daemon prio=5 tid=276 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.hadoop.hbase.regionserver.RemoteProcedureResultReporter.run(RemoteProcedureResultReporter.java:75) "nioEventLoopGroup-2-1" prio=10 tid=92 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) at app//io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) at app//io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) at app//io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:994) at app//io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server handler 1 on default port 44407" daemon prio=5 tid=65 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "pool-11-thread-1" prio=5 tid=69 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.replication.FPBQ.Fifo.handler=2,queue=0,port=36173" daemon prio=5 tid=181 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "NIOWorkerThread-7" daemon prio=5 tid=163 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RS-EventLoopGroup-3-2" daemon prio=10 tid=234 runnable java.lang.Thread.State: RUNNABLE at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait0(Native Method) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:193) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:304) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:368) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "HBase-Metrics2-1" daemon prio=5 tid=153 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "PEWorker-1" daemon prio=5 tid=245 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149) at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2128) "org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber@262e079b" daemon prio=5 tid=74 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.server.namenode.FSNamesystem$LazyPersistFileScrubber.run(FSNamesystem.java:4689) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=36173" daemon prio=5 tid=174 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "IPC Server handler 3 on default port 44407" daemon prio=5 tid=67 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "IPC Server listener on 0" daemon prio=5 tid=94 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hadoop.ipc.Server$Listener.run(Server.java:1559) "FsDatasetAsyncDiskServiceFixer" daemon prio=5 tid=136 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hbase.HBaseTestingUtility$FsDatasetAsyncDiskServiceFixer.run(HBaseTestingUtility.java:599) "regionserver/354fa2a6db59:0.Chore.1" daemon prio=5 tid=273 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-10" daemon prio=5 tid=166 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Idle-Rpc-Conn-Sweeper-pool-0" daemon prio=5 tid=242 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/cluster_b0a2b79c-cac7-9f39-3098-f01052b75b38/dfs/data/data1)" daemon prio=5 tid=117 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:656) "main" prio=5 tid=1 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/java.lang.Thread.dumpThreads(Native Method) at java.base@17.0.11/java.lang.Thread.getAllStackTraces(Thread.java:1671) at app//org.apache.hadoop.hbase.TimedOutTestsListener.buildThreadDump(TimedOutTestsListener.java:92) at app//org.apache.hadoop.hbase.TimedOutTestsListener.buildThreadDiagnosticString(TimedOutTestsListener.java:78) at app//org.apache.hadoop.hbase.TimedOutTestsListener.testFailure(TimedOutTestsListener.java:65) at app//org.junit.runner.notification.SynchronizedRunListener.testFailure(SynchronizedRunListener.java:94) at app//org.junit.runner.notification.RunNotifier$6.notifyListener(RunNotifier.java:177) at app//org.junit.runner.notification.RunNotifier$SafeNotifier.run(RunNotifier.java:72) at app//org.junit.runner.notification.RunNotifier.fireTestFailures(RunNotifier.java:173) at app//org.junit.runner.notification.RunNotifier.fireTestFailure(RunNotifier.java:167) at app//org.apache.maven.surefire.common.junit4.Notifier.fireTestFailure(Notifier.java:100) at app//org.junit.internal.runners.model.EachTestNotifier.addFailure(EachTestNotifier.java:23) at app//org.junit.internal.runners.model.EachTestNotifier.addMultipleFailureException(EachTestNotifier.java:29) at app//org.junit.internal.runners.model.EachTestNotifier.addFailure(EachTestNotifier.java:21) at app//org.junit.runners.ParentRunner.run(ParentRunner.java:419) at app//org.apache.maven.surefire.junit4.JUnit4Provider.execute(JUnit4Provider.java:316) at app//org.apache.maven.surefire.junit4.JUnit4Provider.executeWithRerun(JUnit4Provider.java:240) at app//org.apache.maven.surefire.junit4.JUnit4Provider.executeTestSet(JUnit4Provider.java:214) at app//org.apache.maven.surefire.junit4.JUnit4Provider.invoke(JUnit4Provider.java:155) at app//org.apache.maven.surefire.booter.ForkedBooter.runSuitesInProcess(ForkedBooter.java:385) at app//org.apache.maven.surefire.booter.ForkedBooter.execute(ForkedBooter.java:162) at app//org.apache.maven.surefire.booter.ForkedBooter.run(ForkedBooter.java:507) at app//org.apache.maven.surefire.booter.ForkedBooter.main(ForkedBooter.java:495) "IPC Server handler 2 on default port 44407" daemon prio=5 tid=66 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "qtp1312628392-42" daemon prio=5 tid=42 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RegionServerTracker-0" daemon prio=5 tid=268 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server idle connection scanner for port 45791" daemon prio=5 tid=96 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) at java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) "zk-event-processor-pool-0" daemon prio=5 tid=160 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.replication.FPBQ.Fifo.handler=1,queue=0,port=36173" daemon prio=5 tid=180 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "AsyncFSWAL-0-hdfs://localhost:44407/user/jenkins/test-data/17df60f8-12a3-67d7-a3b4-6fc32f3ae300-prefix:354fa2a6db59,40009,1731384526494" daemon prio=5 tid=284 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Session-HouseKeeper-10572911-1" prio=5 tid=91 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "org.apache.hadoop.hdfs.PeerCache@b94292b" daemon prio=5 tid=215 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.PeerCache.run(PeerCache.java:253) at app//org.apache.hadoop.hdfs.PeerCache.access$000(PeerCache.java:46) at app//org.apache.hadoop.hdfs.PeerCache$1.run(PeerCache.java:124) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Time-limited test" daemon prio=5 tid=22 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.CompletableFuture$Signaller.block(CompletableFuture.java:1864) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.CompletableFuture.waitingGet(CompletableFuture.java:1898) at java.base@17.0.11/java.util.concurrent.CompletableFuture.join(CompletableFuture.java:2117) at app//org.apache.hadoop.hbase.client.TestAsyncReplicationAdminApi.clearPeerAndQueues(TestAsyncReplicationAdminApi.java:96) at java.base@17.0.11/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base@17.0.11/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base@17.0.11/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base@17.0.11/java.lang.reflect.Method.invoke(Method.java:568) at app//org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at app//org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at app//org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at app//org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at app//org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at app//org.junit.rules.TestWatcher$1.evaluate(TestWatcher.java:61) at app//org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at app//org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at app//org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at app//org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at app//org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at app//org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at app//org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at app//org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at app//org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at app//org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at app//org.junit.runners.ParentRunner.run(ParentRunner.java:413) at app//org.junit.runners.Suite.runChild(Suite.java:128) at app//org.junit.runners.Suite.runChild(Suite.java:27) at app//org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at app//org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at app//org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at app//org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at app//org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at app//org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at app//org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at app//org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at app//org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base@17.0.11/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RPCClient-NioEventLoopGroup-4-2" daemon prio=5 tid=328 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) at app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=36173" daemon prio=5 tid=176 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:71) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "RS_COMPACTED_FILES_DISCHARGER-regionserver/354fa2a6db59:0-0" daemon prio=5 tid=747 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-13" daemon prio=5 tid=169 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-16" daemon prio=5 tid=172 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Parameter Sending Thread for localhost/127.0.0.1:44407" daemon prio=5 tid=110 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.SynchronousQueue$TransferQueue.transfer(SynchronousQueue.java:704) at java.base@17.0.11/java.util.concurrent.SynchronousQueue.poll(SynchronousQueue.java:903) at app//org.apache.hadoop.ipc.Client$Connection$RpcRequestSender.run(Client.java:1121) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=40009" daemon prio=5 tid=200 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:71) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "BP-1580922183-172.17.0.2-1731384522326 heartbeating to localhost/127.0.0.1:44407" daemon prio=5 tid=102 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.hdfs.server.datanode.IncrementalBlockReportManager.waitTillNextIBR(IncrementalBlockReportManager.java:158) at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor.offerService(BPServiceActor.java:771) at app//org.apache.hadoop.hdfs.server.datanode.BPServiceActor.run(BPServiceActor.java:914) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Reference Handler" daemon prio=10 tid=2 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/java.lang.ref.Reference.waitForReferencePendingList(Native Method) at java.base@17.0.11/java.lang.ref.Reference.processPendingReferences(Reference.java:253) at java.base@17.0.11/java.lang.ref.Reference$ReferenceHandler.run(Reference.java:215) "qtp764466342-89" daemon prio=5 tid=89 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller@7a30b880" daemon prio=5 tid=73 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.server.namenode.FSNamesystem$NameNodeEditLogRoller.run(FSNamesystem.java:4592) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "pool-14-thread-1" daemon prio=5 tid=132 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1170) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "MemStoreFlusher.0" daemon prio=5 tid=277 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:77) at app//org.apache.hadoop.hbase.regionserver.MemStoreFlusher$FlushHandler.run(MemStoreFlusher.java:322) "RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40009" daemon prio=5 tid=197 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "IPC Client (1688336191) connection to localhost/127.0.0.1:44407 from jenkins" daemon prio=5 tid=109 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.ipc.Client$Connection.waitForWork(Client.java:1042) at app//org.apache.hadoop.ipc.Client$Connection.run(Client.java:1093) "Time-limited test-EventThread" daemon prio=5 tid=193 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) "IPC Server handler 0 on default port 44407" daemon prio=5 tid=64 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "qtp764466342-90" daemon prio=5 tid=90 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.metaPriority.FPBQ.Fifo.handler=0,queue=0,port=36173" daemon prio=5 tid=182 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "pool-6-thread-1" prio=5 tid=46 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "FSEditLogAsync" daemon prio=5 tid=53 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420) at app//org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.dequeueEdit(FSEditLogAsync.java:241) at app//org.apache.hadoop.hdfs.server.namenode.FSEditLogAsync.run(FSEditLogAsync.java:250) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "regionserver/354fa2a6db59:0.logRoller" daemon prio=5 tid=275 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:179) "RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40009" daemon prio=5 tid=195 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "IPC Server handler 4 on default port 45791" daemon prio=5 tid=108 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "Socket Reader #1 for port 0" daemon prio=5 tid=55 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1497) at app//org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1476) "pool-17-thread-1" prio=5 tid=86 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "master/354fa2a6db59:0:becomeActiveMaster-MemStoreChunkPool Statistics" daemon prio=5 tid=213 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "pool-1-thread-1" daemon prio=5 tid=14 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollPort$EventHandlerTask.poll(EPollPort.java:200) at java.base@17.0.11/sun.nio.ch.EPollPort$EventHandlerTask.run(EPollPort.java:281) at java.base@17.0.11/sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:113) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "normalizer-worker-0" daemon prio=5 tid=252 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at app//org.apache.hadoop.hbase.master.normalizer.RegionNormalizerWorkQueue.take(RegionNormalizerWorkQueue.java:146) at app//org.apache.hadoop.hbase.master.normalizer.RegionNormalizerWorker.run(RegionNormalizerWorker.java:191) at java.base@17.0.11/java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:539) at java.base@17.0.11/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "VolumeScannerThread(/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/cluster_b0a2b79c-cac7-9f39-3098-f01052b75b38/dfs/data/data2)" daemon prio=5 tid=118 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.hdfs.server.datanode.VolumeScanner.run(VolumeScanner.java:656) "master:store-WAL-Roller" daemon prio=5 tid=225 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.hbase.wal.AbstractWALRoller.run(AbstractWALRoller.java:179) "NIOWorkerThread-2" daemon prio=5 tid=157 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "NIOWorkerThread-3" daemon prio=5 tid=158 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "PacketResponder: BP-1580922183-172.17.0.2-1731384522326:blk_1073741834_1010, type=LAST_IN_PIPELINE" daemon prio=5 tid=296 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.lang.Object.wait(Object.java:338) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "zk-event-processor-pool-0" daemon prio=5 tid=194 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "pool-1-thread-2" daemon prio=5 tid=15 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420) at java.base@17.0.11/sun.nio.ch.EPollPort$EventHandlerTask.run(EPollPort.java:275) at java.base@17.0.11/sun.nio.ch.AsynchronousChannelGroupImpl$1.run(AsynchronousChannelGroupImpl.java:113) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "GcTimeMonitor obsWindow = 60000, sleepInterval = 5000, maxGcTimePerc = 100" daemon prio=5 tid=35 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.util.GcTimeMonitor.run(GcTimeMonitor.java:161) "MarkedDeleteBlockScrubberThread" daemon prio=5 tid=48 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$MarkedDeleteBlockScrubber.run(BlockManager.java:5326) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ReadOnlyZKClient-127.0.0.1:50727@0x53d9c032" daemon prio=5 tid=320 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$760/0x00007f7288b6e298.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Socket Reader #1 for port 0" daemon prio=5 tid=95 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.hadoop.ipc.Server$Listener$Reader.doRunLoop(Server.java:1497) at app//org.apache.hadoop.ipc.Server$Listener$Reader.run(Server.java:1476) "DataXceiver for client DFSClient_NONMAPREDUCE_1244051054_22 at /127.0.0.1:39600 [Receiving block BP-1580922183-172.17.0.2-1731384522326:blk_1073741832_1008]" daemon prio=5 tid=282 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) at app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) at java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) at java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) at java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) at java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) at app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "org.apache.hadoop.util.JvmPauseMonitor$Monitor@43516cc3" daemon prio=5 tid=93 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:189) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "qtp1312628392-43" daemon prio=5 tid=43 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server handler 4 on default port 44407" daemon prio=5 tid=68 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "org.apache.hadoop.util.JvmPauseMonitor$Monitor@7eb48b85" daemon prio=5 tid=34 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.util.JvmPauseMonitor$Monitor.run(JvmPauseMonitor.java:189) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "IPC Server handler 2 on default port 45791" daemon prio=5 tid=106 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "NIOWorkerThread-12" daemon prio=5 tid=168 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=36173" daemon prio=5 tid=173 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner" daemon prio=5 tid=23 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:176) at app//org.apache.hadoop.fs.FileSystem$Statistics$StatisticsDataReferenceCleaner.run(FileSystem.java:4171) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "snapshot-hfile-cleaner-cache-refresher" daemon prio=5 tid=262 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) at java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) "refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-2/hbase-server/target/test-data/11b2527b-1867-4adf-0240-936d60d7ee1b/cluster_b0a2b79c-cac7-9f39-3098-f01052b75b38/dfs/data/data1/current/BP-1580922183-172.17.0.2-1731384522326" daemon prio=5 tid=123 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.fs.CachingGetSpaceUsed$RefreshThread.run(CachingGetSpaceUsed.java:225) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor@b364b1a" daemon prio=5 tid=49 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hadoop.hdfs.server.blockmanagement.HeartbeatManager$Monitor.run(HeartbeatManager.java:563) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "ProcedureDispatcherTimeoutThread" daemon prio=5 tid=250 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.procedure2.util.DelayedUtil.takeWithoutInterrupt(DelayedUtil.java:81) at app//org.apache.hadoop.hbase.procedure2.RemoteProcedureDispatcher$TimeoutExecutorThread.run(RemoteProcedureDispatcher.java:328) "master/354fa2a6db59:0.Chore.1" daemon prio=5 tid=260 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Block report processor" daemon prio=5 tid=51 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.ArrayBlockingQueue.take(ArrayBlockingQueue.java:420) at app//org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$BlockReportProcessingThread.processQueue(BlockManager.java:5627) at app//org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$BlockReportProcessingThread.run(BlockManager.java:5614) "Hadoop-Metrics-Updater-0" daemon prio=5 tid=98 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40009" daemon prio=5 tid=196 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "PacketResponder: BP-1580922183-172.17.0.2-1731384522326:blk_1073741832_1008, type=LAST_IN_PIPELINE" daemon prio=5 tid=283 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.lang.Object.wait(Object.java:338) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.waitForAckHead(BlockReceiver.java:1367) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver$PacketResponder.run(BlockReceiver.java:1439) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "Common-Cleaner" daemon prio=8 tid=12 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at java.base@17.0.11/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) at java.base@17.0.11/jdk.internal.ref.CleanerImpl.run(CleanerImpl.java:140) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) at java.base@17.0.11/jdk.internal.misc.InnocuousThread.run(InnocuousThread.java:162) "NIOWorkerThread-5" daemon prio=5 tid=161 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RS-EventLoopGroup-1-2" daemon prio=10 tid=258 runnable java.lang.Thread.State: RUNNABLE at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "MobFileCache #0" daemon prio=5 tid=191 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "M:0;354fa2a6db59:36173" daemon prio=5 tid=184 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Object.wait(Native Method) at app//org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:81) at app//org.apache.hadoop.hbase.util.Sleeper.sleep(Sleeper.java:64) at app//org.apache.hadoop.hbase.master.HMaster.waitForMasterActive(HMaster.java:702) at app//org.apache.hadoop.hbase.regionserver.HRegionServer.initializeZooKeeper(HRegionServer.java:1014) at app//org.apache.hadoop.hbase.regionserver.HRegionServer.preRegistrationInitialization(HRegionServer.java:954) at app//org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:1075) at app//org.apache.hadoop.hbase.master.HMaster.run(HMaster.java:603) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.replication.FPBQ.Fifo.handler=0,queue=0,port=40009" daemon prio=5 tid=201 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "qtp764466342-87" daemon prio=5 tid=87 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.eclipse.jetty.io.ManagedSelector.nioSelect(ManagedSelector.java:183) at app//org.eclipse.jetty.io.ManagedSelector.select(ManagedSelector.java:190) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.select(ManagedSelector.java:606) at app//org.eclipse.jetty.io.ManagedSelector$SelectorProducer.produce(ManagedSelector.java:543) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produceTask(EatWhatYouKill.java:362) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:186) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) at app//org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.produce(EatWhatYouKill.java:137) at app//org.eclipse.jetty.io.ManagedSelector$$Lambda$257/0x00007f7288423bb8.run(Unknown Source) at app//org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RS-EventLoopGroup-1-3" daemon prio=10 tid=329 runnable java.lang.Thread.State: RUNNABLE at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native Method) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:220) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:213) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWaitNoTimerChange(EpollEventLoop.java:308) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:365) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "qtp1312628392-44" daemon prio=5 tid=44 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.eclipse.jetty.util.BlockingArrayQueue.poll(BlockingArrayQueue.java:382) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.idleJobPoll(QueuedThreadPool.java:974) at app//org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1018) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "SyncThread:0" daemon prio=5 tid=142 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.zookeeper.server.SyncRequestProcessor.run(SyncRequestProcessor.java:170) "RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=40009" daemon prio=5 tid=199 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.hadoop.hbase.ipc.RpcHandler.getCallRunner(RpcHandler.java:71) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "NIOServerCxnFactory.AcceptThread:localhost/127.0.0.1:50727" daemon prio=5 tid=140 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) at app//org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.select(NIOServerCnxnFactory.java:205) at app//org.apache.zookeeper.server.NIOServerCnxnFactory$AcceptThread.run(NIOServerCnxnFactory.java:181) "NIOWorkerThread-11" daemon prio=5 tid=167 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RS-EventLoopGroup-3-1" daemon prio=10 tid=186 runnable java.lang.Thread.State: RUNNABLE at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait0(Native Method) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.Native.epollWait(Native.java:193) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.epollWait(EpollEventLoop.java:304) at app//org.apache.hbase.thirdparty.io.netty.channel.epoll.EpollEventLoop.run(EpollEventLoop.java:368) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) at app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) at app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RS_REFRESH_PEER-regionserver/354fa2a6db59:0-0" daemon prio=5 tid=334 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcClient-timer-pool-0" daemon prio=5 tid=241 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.waitForNextTick(HashedWheelTimer.java:598) at app//org.apache.hbase.thirdparty.io.netty.util.HashedWheelTimer$Worker.run(HashedWheelTimer.java:494) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "PEWorker-4" daemon prio=5 tid=248 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149) at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2128) "Time-limited test-EventThread" daemon prio=5 tid=404 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at app//org.apache.zookeeper.ClientCnxn$EventThread.run(ClientCnxn.java:550) "RpcServer.replication.FPBQ.Fifo.handler=0,queue=0,port=36173" daemon prio=5 tid=179 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "CacheReplicationMonitor(1826937612)" daemon prio=5 tid=75 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1759) at app//org.apache.hadoop.hdfs.server.blockmanagement.CacheReplicationMonitor.run(CacheReplicationMonitor.java:186) "IPC Server Responder" daemon prio=5 tid=97 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.hadoop.ipc.Server$Responder.doRunLoop(Server.java:1733) at app//org.apache.hadoop.ipc.Server$Responder.run(Server.java:1716) "ReadOnlyZKClient-127.0.0.1:50727@0x502fe4e9" daemon prio=5 tid=325 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.DelayQueue.poll(DelayQueue.java:265) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient.run(ReadOnlyZKClient.java:370) at app//org.apache.hadoop.hbase.zookeeper.ReadOnlyZKClient$$Lambda$760/0x00007f7288b6e298.run(Unknown Source) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "DataXceiver for client DFSClient_NONMAPREDUCE_-32584894_22 at /127.0.0.1:45502 [Receiving block BP-1580922183-172.17.0.2-1731384522326:blk_1073741840_1016]" daemon prio=5 tid=792 runnable java.lang.Thread.State: RUNNABLE at java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) at java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) at java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) at java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) at app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) at app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) at app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) at java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) at java.base@17.0.11/java.io.BufferedInputStream.read1(BufferedInputStream.java:284) at java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:343) at java.base@17.0.11/java.io.DataInputStream.read(DataInputStream.java:151) at app//org.apache.hadoop.io.IOUtils.readFully(IOUtils.java:214) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doReadFully(PacketReceiver.java:221) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.doRead(PacketReceiver.java:144) at app//org.apache.hadoop.hdfs.protocol.datatransfer.PacketReceiver.receiveNextPacket(PacketReceiver.java:119) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receivePacket(BlockReceiver.java:553) at app//org.apache.hadoop.hdfs.server.datanode.BlockReceiver.receiveBlock(BlockReceiver.java:1011) at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.writeBlock(DataXceiver.java:920) at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.opWriteBlock(Receiver.java:176) at app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.processOp(Receiver.java:110) at app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:299) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=36173" daemon prio=5 tid=175 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquire(AbstractQueuedSynchronizer.java:715) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer.acquireSharedInterruptibly(AbstractQueuedSynchronizer.java:1047) at java.base@17.0.11/java.util.concurrent.Semaphore.acquire(Semaphore.java:318) at app//org.apache.hadoop.hbase.ipc.FastPathRpcHandler.getCallRunner(FastPathRpcHandler.java:55) at app//org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:85) "IPC Server handler 0 on default port 45791" daemon prio=5 tid=104 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) at app//org.apache.hadoop.ipc.CallQueueManager.take(CallQueueManager.java:370) at app//org.apache.hadoop.ipc.Server$Handler.run(Server.java:3165) "RedundancyMonitor" daemon prio=5 tid=47 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/java.lang.Thread.sleep(Native Method) at java.base@17.0.11/java.lang.Thread.sleep(Thread.java:344) at java.base@17.0.11/java.util.concurrent.TimeUnit.sleep(TimeUnit.java:446) at app//org.apache.hadoop.hdfs.server.blockmanagement.BlockManager$RedundancyMonitor.run(BlockManager.java:5352) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "RS_OPEN_PRIORITY_REGION-regionserver/354fa2a6db59:0-0" daemon prio=5 tid=312 in Object.wait() java.lang.Thread.State: WAITING (on object monitor) at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) at java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) at java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "pool-5-thread-1" prio=5 tid=36 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:1182) at java.base@17.0.11/java.util.concurrent.ScheduledThreadPoolExecutor$DelayedWorkQueue.take(ScheduledThreadPoolExecutor.java:899) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) at java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) at java.base@17.0.11/java.lang.Thread.run(Thread.java:840) "PEWorker-5" daemon prio=5 tid=249 timed_waiting java.lang.Thread.State: TIMED_WAITING at java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) at java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) at java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:167) at app//org.apache.hadoop.hbase.procedure2.AbstractProcedureScheduler.poll(AbstractProcedureScheduler.java:149) at app//org.apache.hadoop.hbase.procedure2.ProcedureExecutor$WorkerThread.run(ProcedureExecutor.java:2128)